diff options
Diffstat (limited to 'usr/src/test/zfs-tests')
24 files changed, 1529 insertions, 0 deletions
diff --git a/usr/src/test/zfs-tests/callbacks/zfs_mmp.ksh b/usr/src/test/zfs-tests/callbacks/zfs_mmp.ksh new file mode 100755 index 0000000000..df2cd132d3 --- /dev/null +++ b/usr/src/test/zfs-tests/callbacks/zfs_mmp.ksh @@ -0,0 +1,37 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security. +# All rights reserved. +# + +# $1: number of lines to output (default: 40) +typeset lines=${1:-40} +typeset history=$(cat /sys/module/zfs/parameters/zfs_multihost_history) + +if [ $history -eq 0 ]; then + exit +fi + +for f in /proc/spl/kstat/zfs/*/multihost; do + echo "=================================================================" + echo " Last $lines lines of $f" + echo "=================================================================" + + sudo tail -n $lines $f +done + +echo "=================================================================" +echo " End of zfs multihost log" +echo "=================================================================" diff --git a/usr/src/test/zfs-tests/include/commands.cfg b/usr/src/test/zfs-tests/include/commands.cfg index 3b21caafb6..e5d8dbe403 100644 --- a/usr/src/test/zfs-tests/include/commands.cfg +++ b/usr/src/test/zfs-tests/include/commands.cfg @@ -56,6 +56,7 @@ export USR_BIN_FILES='awk getent grep head + hostid hostname id iostat diff --git a/usr/src/test/zfs-tests/include/libtest.shlib b/usr/src/test/zfs-tests/include/libtest.shlib index 0da7d929f1..295620102a 100644 --- a/usr/src/test/zfs-tests/include/libtest.shlib +++ b/usr/src/test/zfs-tests/include/libtest.shlib @@ -2703,3 +2703,89 @@ function mdb_ctf_set_int return 0 } + +# +# Set a global system tunable (64-bit value) +# +# $1 tunable name +# $2 tunable values +# +function set_tunable64 +{ + set_tunable_impl "$1" "$2" Z +} + +# +# Set a global system tunable (32-bit value) +# +# $1 tunable name +# $2 tunable values +# +function set_tunable32 +{ + set_tunable_impl "$1" "$2" W +} + +function set_tunable_impl +{ + typeset tunable="$1" + typeset value="$2" + typeset mdb_cmd="$3" + typeset module="${4:-zfs}" + + [[ -z "$tunable" ]] && return 1 + [[ -z "$value" ]] && return 1 + [[ -z "$mdb_cmd" ]] && return 1 + + case "$(uname)" in + Linux) + typeset zfs_tunables="/sys/module/$module/parameters" + [[ -w "$zfs_tunables/$tunable" ]] || return 1 + echo -n "$value" > "$zfs_tunables/$tunable" + return "$?" + ;; + SunOS) + [[ "$module" -eq "zfs" ]] || return 1 + echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw + return "$?" + ;; + esac +} + +# +# Get a global system tunable +# +# $1 tunable name +# +function get_tunable +{ + get_tunable_impl "$1" +} + +function get_tunable_impl +{ + typeset tunable="$1" + typeset module="${2:-zfs}" + + [[ -z "$tunable" ]] && return 1 + + case "$(uname)" in + Linux) + typeset zfs_tunables="/sys/module/$module/parameters" + [[ -f "$zfs_tunables/$tunable" ]] || return 1 + cat $zfs_tunables/$tunable + return "$?" + ;; + SunOS) + typeset value=$(mdb -k -e "$tunable/X | ::eval .=U") + if [[ $? -ne 0 ]]; then + log_fail "Failed to get value of '$tunable' from mdb." + return 1 + fi + echo $value + return 0 + ;; + esac + + return 1 +} diff --git a/usr/src/test/zfs-tests/runfiles/delphix.run b/usr/src/test/zfs-tests/runfiles/delphix.run index ed3961f323..41e6d30e8b 100644 --- a/usr/src/test/zfs-tests/runfiles/delphix.run +++ b/usr/src/test/zfs-tests/runfiles/delphix.run @@ -462,6 +462,12 @@ tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos', [/opt/zfs-tests/tests/functional/mmap] tests = ['mmap_read_001_pos', 'mmap_write_001_pos'] +[/opt/zfs-tests/tests/functional/mmp] +tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval', + 'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import', + 'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history', + 'mmp_on_zdb', 'mmp_write_distribution'] + [/opt/zfs-tests/tests/functional/mount] tests = ['umount_001', 'umountall_001'] diff --git a/usr/src/test/zfs-tests/runfiles/omnios.run b/usr/src/test/zfs-tests/runfiles/omnios.run index 5847e0e980..e46c59416e 100644 --- a/usr/src/test/zfs-tests/runfiles/omnios.run +++ b/usr/src/test/zfs-tests/runfiles/omnios.run @@ -431,6 +431,12 @@ tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos', [/opt/zfs-tests/tests/functional/mmap] tests = ['mmap_read_001_pos', 'mmap_write_001_pos'] +[/opt/zfs-tests/tests/functional/mmp] +tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval', + 'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import', + 'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history', + 'mmp_on_zdb', 'mmp_write_distribution'] + [/opt/zfs-tests/tests/functional/mount] tests = ['umount_001', 'umountall_001'] diff --git a/usr/src/test/zfs-tests/runfiles/openindiana.run b/usr/src/test/zfs-tests/runfiles/openindiana.run index 3c1202f751..188f9b24c2 100644 --- a/usr/src/test/zfs-tests/runfiles/openindiana.run +++ b/usr/src/test/zfs-tests/runfiles/openindiana.run @@ -431,6 +431,12 @@ tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos', [/opt/zfs-tests/tests/functional/mmap] tests = ['mmap_read_001_pos', 'mmap_write_001_pos'] +[/opt/zfs-tests/tests/functional/mmp] +tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval', + 'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import', + 'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history', + 'mmp_on_zdb', 'mmp_write_distribution'] + [/opt/zfs-tests/tests/functional/mount] tests = ['umount_001', 'umountall_001'] diff --git a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index ebf4e77ba8..90e723a8cc 100644 --- a/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/usr/src/test/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -56,6 +56,7 @@ typeset -a properties=( "leaked" "bootsize" "checkpoint" + "multihost" "feature@async_destroy" "feature@empty_bpobj" "feature@lz4_compress" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/Makefile b/usr/src/test/zfs-tests/tests/functional/mmp/Makefile new file mode 100644 index 0000000000..7b26281882 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/Makefile @@ -0,0 +1,21 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2019 Joyent, Inc. +# + +include $(SRC)/Makefile.master + +ROOTOPTPKG = $(ROOT)/opt/zfs-tests +TARGETDIR = $(ROOTOPTPKG)/tests/functional/mmp + +include $(SRC)/test/zfs-tests/Makefile.com diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/cleanup.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/cleanup.ksh new file mode 100755 index 0000000000..82f06567cd --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/cleanup.ksh @@ -0,0 +1,36 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "global" + +case "$(uname)" in +SunOS) h=$(cat /var/tmp/zfs_test_hostid.txt) + mmp_set_hostid $h + rm /var/tmp/zfs_test_hostid.txt + ;; +esac +log_must set_tunable64 zfs_multihost_history 0 + +log_pass "mmp cleanup passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp.cfg b/usr/src/test/zfs-tests/tests/functional/mmp/mmp.cfg new file mode 100644 index 0000000000..52680c275a --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp.cfg @@ -0,0 +1,40 @@ +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +export PREV_UBER="$TEST_BASE_DIR/mmp-uber-prev.txt" +export CURR_UBER="$TEST_BASE_DIR/mmp-uber-curr.txt" +export DISK=${DISKS%% *} + +export HOSTID_FILE="/etc/hostid" +export HOSTID1=01234567 +export HOSTID2=89abcdef + +export TXG_TIMEOUT_LONG=5000 +export TXG_TIMEOUT_DEFAULT=5 + +export MMP_POOL=mmppool +export MMP_DIR=$TEST_BASE_DIR/mmp +export MMP_CACHE=$MMP_DIR/zpool.cache +export MMP_ZTEST_LOG=$MMP_DIR/ztest.log +export MMP_HISTORY=100 +export MMP_HISTORY_OFF=0 + +export MMP_INTERVAL_HOUR=$((60*60*1000)) +export MMP_INTERVAL_DEFAULT=1000 +export MMP_INTERVAL_MIN=100 diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp.kshlib b/usr/src/test/zfs-tests/tests/functional/mmp/mmp.kshlib new file mode 100644 index 0000000000..0dc255998b --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp.kshlib @@ -0,0 +1,292 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Use is subject to license terms. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg + + +function check_pool_import # pool opts token keyword +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + typeset token=$3 + typeset keyword=$4 + + zpool import $opts 2>&1 | \ + nawk -v token="$token:" '($1==token) {print $0}' | \ + grep -i "$keyword" > /dev/null 2>&1 + + return $? +} + +function is_pool_imported # pool opts +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + + check_pool_import "$pool" "$opts" "status" \ + "The pool is currently imported" + return $? +} + +function wait_pool_imported # pool opts +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + + while is_pool_imported "$pool" "$opts"; do + log_must sleep 5 + done + + return 0 +} + +function try_pool_import # pool opts message +{ + typeset pool=${1:-$MMP_POOL} + typeset opts=$2 + typeset msg=$3 + + zpool import $opts $pool 2>&1 | grep -i "$msg" + + return $? +} + +function chr2ascii +{ + case "$1" in + 0) asc="30";; + 1) asc="31";; + 2) asc="32";; + 3) asc="33";; + 4) asc="34";; + 5) asc="35";; + 6) asc="36";; + 7) asc="37";; + 8) asc="38";; + 9) asc="39";; + a) asc="61";; + b) asc="62";; + c) asc="63";; + d) asc="64";; + e) asc="65";; + f) asc="66";; + esac +} + +function mmp_set_hostid +{ + typeset hostid=$1 + + case "$(uname)" in + Linux) + a=${hostid:6:2} + b=${hostid:4:2} + c=${hostid:2:2} + d=${hostid:0:2} + + printf "\\x$a\\x$b\\x$c\\x$d" >$HOSTID_FILE + + if [ $(hostid) != "$hostid" ]; then + return 1 + fi + ;; + SunOS) + # + # Given a hostid in hex, we have to convert to decimal, then + # save the ascii string representation in the kernel. The + # 'hostid' command will get the decimal SI_HW_SERIAL value via + # sysinfo, then print that as an 8 digit hex number. + # + typeset dec=$(mdb -e "$hostid=E" | sed -e 's/ *//g') + typeset len=$(echo $dec | awk '{print length($0)}') + if [[ $len -lt 0 || $len -gt 10 ]]; then + return + fi + typeset pos=0 + while [[ $pos -lt $len ]]; do + chr2ascii ${dec:$pos:1} + echo "hw_serial+${pos}/v $asc" | mdb -kw >/dev/null 2>&1 + pos=$(($pos + 1)) + done + echo "hw_serial+${pos}/v 0" | mdb -kw >/dev/null 2>&1 + ;; + esac + + return 0 +} + +function mmp_clear_hostid +{ + case "$(uname)" in + Linux) rm -f $HOSTID_FILE;; + SunOS) mmp_set_hostid "00000000";; + esac +} + +function mmp_pool_create_simple # pool dir +{ + typeset pool=${1:-$MMP_POOL} + typeset dir=${2:-$MMP_DIR} + + log_must mkdir -p $dir + log_must rm -f $dir/* + log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 + + log_must mmp_set_hostid $HOSTID1 + log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ + mirror $dir/vdev1 $dir/vdev2 + log_must zpool set multihost=on $pool +} + +function mmp_pool_create # pool dir +{ + typeset pool=${1:-$MMP_POOL} + typeset dir=${2:-$MMP_DIR} + typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" + + mmp_pool_create_simple $pool $dir + + log_must mv $MMP_CACHE ${MMP_CACHE}.stale + log_must zpool export $pool + log_must mmp_set_hostid $HOSTID2 + + log_note "Starting ztest in the background as hostid $HOSTID1" + log_must eval "ZFS_HOSTID=$HOSTID1 /usr/bin/ztest $opts >$MMP_ZTEST_LOG 2>&1 &" + + while ! is_pool_imported "$pool" "-d $dir"; do + log_must pgrep ztest + log_must sleep 5 + done +} + +function mmp_pool_destroy # pool dir +{ + typeset pool=${1:-$MMP_POOL} + typeset dir=${2:-$MMP_DIR} + + ZTESTPID=$(pgrep ztest) + if [ -n "$ZTESTPID" ]; then + log_must kill $ZTESTPID + wait $ZTESTPID + fi + + if poolexists $pool; then + destroy_pool $pool + fi + + log_must rm -f $dir/* + mmp_clear_hostid +} + +function mmp_pool_set_hostid # pool hostid +{ + typeset pool=$1 + typeset hostid=$2 + + log_must mmp_set_hostid $hostid + log_must zpool export $pool + log_must zpool import $pool + + return 0 +} + +# Return the number of seconds the activity check portion of the import process +# will take. Does not include the time to find devices and assemble the +# preliminary pool configuration passed into the kernel. +function seconds_mmp_waits_for_activity +{ + typeset import_intervals=$(get_tunable zfs_multihost_import_intervals) + typeset interval=$(get_tunable zfs_multihost_interval) + typeset seconds=$((interval*import_intervals/1000)) + + echo $seconds +} + +function import_no_activity_check # pool opts +{ + typeset pool=$1 + typeset opts=$2 + + typeset max_duration=$(seconds_mmp_waits_for_activity) + + SECONDS=0 + zpool import $opts $pool + typeset rc=$? + + if [[ $SECONDS -gt $max_duration ]]; then + log_fail "unexpected activity check (${SECONDS}s gt \ +$max_duration)" + fi + + return $rc +} + +function import_activity_check # pool opts +{ + typeset pool=$1 + typeset opts=$2 + + typeset min_duration=$(seconds_mmp_waits_for_activity) + + SECONDS=0 + zpool import $opts $pool + typeset rc=$? + + if [[ $SECONDS -le $min_duration ]]; then + log_fail "expected activity check (${SECONDS}s le \ +$min_duration)" + fi + + return $rc +} + +function clear_mmp_history +{ + log_must set_tunable64 zfs_multihost_history $MMP_HISTORY_OFF + log_must set_tunable64 zfs_multihost_history $MMP_HISTORY +} + +function count_skipped_mmp_writes # pool duration +{ + typeset pool=$1 + typeset -i duration=$2 + typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" + + sleep $duration + awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path" +} + +function count_mmp_writes # pool duration +{ + typeset pool=$1 + typeset -i duration=$2 + typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" + + log_must sleep $duration + awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path" +} diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_active_import.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_active_import.ksh new file mode 100755 index 0000000000..59f1e1ef67 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_active_import.ksh @@ -0,0 +1,119 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +# DESCRIPTION: +# Under no circumstances when multihost is active, should an active pool +# with one hostid be importable by a host with a different hostid. +# +# STRATEGY: +# 1. Simulate an active pool on another host with ztest. +# 2. Verify 'zpool import' reports an active pool. +# 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. +# 4. Kill ztest to make pool eligible for import. +# 5. Verify 'zpool import' fails with the expected error message. +# 6. Verify 'zpool import $MMP_POOL' fails with the expected message. +# 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. +# 8. Verify pool may be exported/imported without -f argument. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + mmp_pool_destroy $MMP_POOL $MMP_DIR + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off active pool activity checks" +log_onexit cleanup + +# 1. Simulate an active pool on another host with ztest. +mmp_pool_destroy $MMP_POOL $MMP_DIR +mmp_pool_create $MMP_POOL $MMP_DIR + +# 2. Verify 'zpool import' reports an active pool. +log_must mmp_set_hostid $HOSTID2 +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must is_pool_imported $MMP_POOL "-d $MMP_DIR" + +# 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. +MMP_IMPORTED_MSG="Cannot import '$MMP_POOL': pool is imported" + +log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG" +for i in {1..10}; do + log_must try_pool_import $MMP_POOL "-f -d $MMP_DIR" "$MMP_IMPORTED_MSG" +done + +log_must try_pool_import $MMP_POOL "-c ${MMP_CACHE}.stale" "$MMP_IMPORTED_MSG" + +for i in {1..10}; do + log_must try_pool_import $MMP_POOL "-f -c ${MMP_CACHE}.stale" \ + "$MMP_IMPORTED_MSG" +done + +# 4. Kill ztest to make pool eligible for import. Poll with 'zpool status'. +ZTESTPID=$(pgrep ztest) +if [ -n "$ZTESTPID" ]; then + log_must kill -9 $ZTESTPID +fi +log_must wait_pool_imported $MMP_POOL "-d $MMP_DIR" + +# 5. Verify 'zpool import' fails with the expected error message, when +# - hostid=0: - configuration error +# - hostid=matches - safe to import the pool +# - hostid=different - previously imported on a different system +# +log_must mmp_clear_hostid +case "$(uname)" in +Linux) MMP_IMPORTED_MSG="Set a unique system hostid";; +SunOS) MMP_IMPORTED_MSG="Check the SMF svc:/system/hostid service.";; +esac +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" "$MMP_IMPORTED_MSG" + +log_must mmp_set_hostid $HOSTID1 +MMP_IMPORTED_MSG="The pool can be imported" +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" "$MMP_IMPORTED_MSG" + +log_must mmp_clear_hostid +log_must mmp_set_hostid $HOSTID2 +MMP_IMPORTED_MSG="The pool was last accessed by another system." +log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "status" "$MMP_IMPORTED_MSG" + +# 6. Verify 'zpool import $MMP_POOL' fails with the expected message. +MMP_IMPORTED_MSG="pool was previously in use from another system." +log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG" + +# 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. +# Default interval results in minimum activity test 10s which +# makes detection of the activity test reliable. +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT +log_must import_activity_check $MMP_POOL "-f -d $MMP_DIR" + +# 8 Verify pool may be exported/imported without -f argument. +log_must zpool export $MMP_POOL +log_must import_no_activity_check $MMP_POOL "-d $MMP_DIR" + +log_pass "multihost=on|off active pool activity checks passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh new file mode 100755 index 0000000000..bf00e4b8d5 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh @@ -0,0 +1,110 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +# DESCRIPTION: +# Verify import behavior for exported pool (no activity check) +# +# STRATEGY: +# 1. Create a zpool +# 2. Verify multihost=off and hostids match (no activity check) +# 3. Verify multihost=off and hostids differ (no activity check) +# 4. Verify multihost=off and hostid zero allowed (no activity check) +# 5. Verify multihost=on and hostids match (no activity check) +# 6. Verify multihost=on and hostids differ (no activity check) +# 7. Verify multihost=on and hostid zero fails (no activity check) +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off activity checks exported pool" +log_onexit cleanup + +# 1. Create a zpool +log_must mmp_set_hostid $HOSTID1 +default_setup_noexit $DISK + +# 2. Verify multihost=off and hostids match (no activity check) +log_must zpool set multihost=off $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 3. Verify multihost=off and hostids differ (no activity check) +for opt in "" "-f"; do + log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 + log_must zpool export $TESTPOOL + log_must mmp_clear_hostid + log_must mmp_set_hostid $HOSTID2 + log_must import_no_activity_check $TESTPOOL $opt +done + +# 4. Verify multihost=off and hostid zero allowed (no activity check) +log_must mmp_clear_hostid + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 5. Verify multihost=on and hostids match (no activity check) +log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 +log_must zpool set multihost=on $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 6. Verify multihost=on and hostids differ (no activity check) +for opt in "" "-f"; do + log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 + log_must zpool export $TESTPOOL + log_must mmp_clear_hostid + log_must mmp_set_hostid $HOSTID2 + log_must import_no_activity_check $TESTPOOL $opt +done + +# 7. Verify multihost=on and hostid zero fails (no activity check) +log_must zpool export $TESTPOOL +log_must mmp_clear_hostid + +for opt in "" "-f"; do + case "$(uname)" in + Linux) MMP_IMPORTED_MSG="Set a unique system hostid";; + SunOS) MMP_IMPORTED_MSG="Check the SMF svc:/system/hostid service.";; + esac + log_must check_pool_import $TESTPOOL "" "action" "$MMP_IMPORTED_MSG" + log_mustnot import_no_activity_check $TESTPOOL $opt +done + +log_pass "multihost=on|off exported pool activity checks passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh new file mode 100755 index 0000000000..59a2a3d09c --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh @@ -0,0 +1,101 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +# DESCRIPTION: +# Verify import behavior for inactive, but not exported, pools +# +# STRATEGY: +# 1. Create a zpool +# 2. Verify multihost=off and hostids match (no activity check) +# 3. Verify multihost=off and hostids differ (no activity check) +# 4. Verify multihost=off and hostid allowed (no activity check) +# 5. Verify multihost=on and hostids match (no activity check) +# 6. Verify multihost=on and hostids differ (activity check) +# 7. Verify multihost=on and hostid zero fails (no activity check) +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must mmp_clear_hostid +} + +log_assert "multihost=on|off inactive pool activity checks" +log_onexit cleanup + +# 1. Create a zpool +log_must mmp_set_hostid $HOSTID1 +default_setup_noexit $DISK + +# 2. Verify multihost=off and hostids match (no activity check) +log_must zpool set multihost=off $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export -F $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 3. Verify multihost=off and hostids differ (no activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +log_must mmp_set_hostid $HOSTID2 +log_mustnot import_no_activity_check $TESTPOOL "" +log_must import_no_activity_check $TESTPOOL "-f" + +# 4. Verify multihost=off and hostid zero allowed (no activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +log_mustnot import_no_activity_check $TESTPOOL "" +log_must import_no_activity_check $TESTPOOL "-f" + +# 5. Verify multihost=on and hostids match (no activity check) +log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 +log_must zpool set multihost=on $TESTPOOL + +for opt in "" "-f"; do + log_must zpool export -F $TESTPOOL + log_must import_no_activity_check $TESTPOOL $opt +done + +# 6. Verify multihost=on and hostids differ (activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +log_must mmp_set_hostid $HOSTID2 +log_mustnot import_activity_check $TESTPOOL "" +log_must import_activity_check $TESTPOOL "-f" + +# 7. Verify multihost=on and hostid zero fails (no activity check) +log_must zpool export -F $TESTPOOL +log_must mmp_clear_hostid +case "$(uname)" in +Linux) MMP_IMPORTED_MSG="Set a unique system hostid";; +SunOS) MMP_IMPORTED_MSG="Check the SMF svc:/system/hostid service.";; +esac +log_must check_pool_import $TESTPOOL "-f" "action" "$MMP_IMPORTED_MSG" +log_mustnot import_no_activity_check $TESTPOOL "-f" + +log_pass "multihost=on|off inactive pool activity checks passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_interval.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_interval.ksh new file mode 100755 index 0000000000..e07677c4f1 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_interval.ksh @@ -0,0 +1,50 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +# DESCRIPTION: +# zfs_multihost_interval should only accept valid values. +# +# STRATEGY: +# 1. Set zfs_multihost_interval to invalid values (negative). +# 2. Set zfs_multihost_interval to valid values. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT +} + +log_assert "zfs_multihost_interval cannot be set to an invalid value" +log_onexit cleanup + +if [[ $(uname) == "Linux" ]]; then + log_mustnot set_tunable64 zfs_multihost_interval -1 +fi +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + +log_pass "zfs_multihost_interval cannot be set to an invalid value" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_off.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_off.ksh new file mode 100755 index 0000000000..8bef86a0ff --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_off.ksh @@ -0,0 +1,79 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# When multihost=off ensure that leaf vdev uberblocks are not updated. +# +# STRATEGY: +# 1. Set multihost=off (disables mmp) +# 2. Set zfs_txg_timeout to large value +# 3. Create a zpool +# 4. Find the current "best" uberblock +# 5. Sleep for enough time for uberblocks to change +# 6. Find the current "best" uberblock +# 7. If the uberblock changed, fail +# 8. Set multihost=on +# 9. Sleep for enough time for uberblocks to change +# 10. Find the current "best" uberblock +# 11. If uberblocks didn't change, fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + log_must rm -f $PREV_UBER $CURR_UBER + log_must mmp_clear_hostid +} + +log_assert "mmp thread won't write uberblocks with multihost=off" +log_onexit cleanup + +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_MIN +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=off $TESTPOOL + +log_must zdb -u $TESTPOOL > $PREV_UBER +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if ! diff "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp thread has updated an uberblock" +fi + +log_must zpool set multihost=on $TESTPOOL +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if diff "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp failed to update uberblocks" +fi + +log_pass "mmp thread won't write uberblocks with multihost=off passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh new file mode 100755 index 0000000000..07384c6231 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Ensure that the MMP thread is writing uberblocks. +# +# STRATEGY: +# 1. Set zfs_txg_timeout to large value +# 2. Create a zpool +# 3. Find the current "best" uberblock +# 4. Sleep for enough time for a potential uberblock update +# 5. Find the current "best" uberblock +# 6. If the uberblock never changed, fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must rm -f $PREV_UBER $CURR_UBER + log_must mmp_clear_hostid +} + +log_assert "mmp thread writes uberblocks (MMP)" +log_onexit cleanup + +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=on $TESTPOOL + +log_must zdb -u $TESTPOOL > $PREV_UBER +log_must sleep 5 +log_must zdb -u $TESTPOOL > $CURR_UBER + +if diff -u "$CURR_UBER" "$PREV_UBER"; then + log_fail "mmp failed to update uberblocks" +fi + +log_pass "mmp thread writes uberblocks (MMP) passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh new file mode 100755 index 0000000000..0cb38f8899 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh @@ -0,0 +1,73 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Ensure that MMP updates uberblocks at the expected intervals. +# +# STRATEGY: +# 1. Set zfs_txg_timeout to large value +# 2. Create a zpool +# 3. Clear multihost history +# 4. Sleep, then collect count of uberblocks written +# 5. If number of changes seen is less than min threshold, then fail +# 6. If number of changes seen is more than max threshold, then fail +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +UBER_CHANGES=0 +EXPECTED=$(($(echo $DISKS | wc -w) * 10)) +FUDGE=$((EXPECTED * 20 / 100)) +MIN=$((EXPECTED - FUDGE)) +MAX=$((EXPECTED + FUDGE)) + +function cleanup +{ + default_cleanup_noexit + set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_DEFAULT + log_must mmp_clear_hostid +} + +log_assert "Ensure MMP uberblocks update at the correct interval" +log_onexit cleanup + +log_must set_tunable64 zfs_txg_timeout $TXG_TIMEOUT_LONG +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit "$DISKS" +log_must zpool set multihost=on $TESTPOOL +clear_mmp_history +UBER_CHANGES=$(count_mmp_writes $TESTPOOL 10) + +log_note "Uberblock changed $UBER_CHANGES times" + +if [ $UBER_CHANGES -lt $MIN ]; then + log_fail "Fewer uberblock writes occured than expected ($EXPECTED)" +fi + +if [ $UBER_CHANGES -gt $MAX ]; then + log_fail "More uberblock writes occured than expected ($EXPECTED)" +fi + +log_pass "Ensure MMP uberblocks update at the correct interval passed" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh new file mode 100755 index 0000000000..131fd21e88 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh @@ -0,0 +1,81 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 Lawrence Livermore National Security, LLC. +# Copyright (c) 2018 by Nutanix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +# +# Description: +# zdb will work while multihost is enabled. +# +# Strategy: +# 1. Create a pool +# 2. Enable multihost +# 3. Run zdb -d with pool and dataset arguments. +# 4. Create a checkpoint +# 5. Run zdb -kd with pool and dataset arguments. +# 6. Discard the checkpoint +# 7. Export the pool +# 8. Run zdb -ed with pool and dataset arguments. +# + +function cleanup +{ + datasetexists $TESTPOOL && destroy_pool $TESTPOOL + for DISK in $DISKS; do + zpool labelclear -f $DEV_RDSKDIR/$DISK + done + log_must mmp_clear_hostid +} + +log_assert "Verify zdb -d works while multihost is enabled" +log_onexit cleanup + +verify_runnable "global" +verify_disk_count "$DISKS" 2 + +default_mirror_setup_noexit $DISKS +log_must mmp_set_hostid $HOSTID1 +log_must zpool set multihost=on $TESTPOOL +log_must zfs snap $TESTPOOL/$TESTFS@snap + +log_must zdb -d $TESTPOOL +log_must zdb -d $TESTPOOL/ +log_must zdb -d $TESTPOOL/$TESTFS +log_must zdb -d $TESTPOOL/$TESTFS@snap + +log_must zpool checkpoint $TESTPOOL +log_must zdb -kd $TESTPOOL +log_must zdb -kd $TESTPOOL/ +log_must zdb -kd $TESTPOOL/$TESTFS +log_must zdb -kd $TESTPOOL/$TESTFS@snap +log_must zpool checkpoint -d $TESTPOOL + +log_must zpool export $TESTPOOL + +log_must zdb -ed $TESTPOOL +log_must zdb -ed $TESTPOOL/ +log_must zdb -ed $TESTPOOL/$TESTFS +log_must zdb -ed $TESTPOOL/$TESTFS@snap + +log_must zpool import $TESTPOOL + +cleanup + +log_pass "zdb -d works while multihost is enabled" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh new file mode 100755 index 0000000000..3c8f00cde9 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Ensure that the MMP thread is notified when zfs_multihost_interval is +# reduced. +# +# STRATEGY: +# 1. Set zfs_multihost_interval to much longer than the test duration +# 2. Create a zpool and enable multihost +# 3. Verify no MMP writes occurred +# 4. Set zfs_multihost_interval to 1 second +# 5. Sleep briefly +# 6. Verify MMP writes began +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + default_cleanup_noexit + log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT + log_must mmp_clear_hostid +} + +log_assert "mmp threads notified when zfs_multihost_interval reduced" +log_onexit cleanup + +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_HOUR +log_must mmp_set_hostid $HOSTID1 + +default_setup_noexit $DISK +log_must zpool set multihost=on $TESTPOOL + +clear_mmp_history +log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT +uber_count=$(count_mmp_writes $TESTPOOL 1) + +if [ $uber_count -eq 0 ]; then + log_fail "mmp writes did not start when zfs_multihost_interval reduced" +fi + +log_pass "mmp threads notified when zfs_multihost_interval reduced" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh new file mode 100755 index 0000000000..7504caa4d1 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh @@ -0,0 +1,92 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Verify MMP writes are distributed evenly among leaves +# +# STRATEGY: +# 1. Create an asymmetric mirrored pool +# 2. Enable multihost and multihost_history +# 3. Delay for MMP writes to occur +# 4. Verify the MMP writes are distributed evenly across leaf vdevs +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + log_must zpool destroy $MMP_POOL + log_must rm $MMP_DIR/file.{0,1,2,3,4,5,6,7} + log_must rm $MMP_HISTORY_TMP + log_must rmdir $MMP_DIR + log_must mmp_clear_hostid +} + +log_assert "mmp writes are evenly distributed across leaf vdevs" +log_onexit cleanup + +MMP_HISTORY_TMP=$MMP_DIR/history +MMP_HISTORY=/proc/spl/kstat/zfs/$MMP_POOL/multihost + +# Step 1 +log_must mkdir -p $MMP_DIR +log_must truncate -s 128M $MMP_DIR/file.{0,1,2,3,4,5,6,7} +log_must zpool create -f $MMP_POOL mirror $MMP_DIR/file.{0,1} mirror $MMP_DIR/file.{2,3,4,5,6,7} + +# Step 2 +log_must mmp_set_hostid $HOSTID1 +log_must zpool set multihost=on $MMP_POOL +set_tunable64 zfs_multihost_history 0 +set_tunable64 zfs_multihost_history 40 + +# Step 3 +# default settings, every leaf written once/second +sleep 4 + +# Step 4 +typeset -i min_writes=999 +typeset -i max_writes=0 +typeset -i write_count +# copy to get as close to a consistent view as possible +cat $MMP_HISTORY > $MMP_HISTORY_TMP +for x in $(seq 0 7); do + write_count=$(grep -c file.${x} $MMP_HISTORY_TMP) + if [ $write_count -lt $min_writes ]; then + min_writes=$write_count + fi + if [ $write_count -gt $max_writes ]; then + max_writes=$write_count + fi +done +log_note "mmp min_writes $min_writes max_writes $max_writes" + +if [ $min_writes -lt 1 ]; then + log_fail "mmp writes were not counted correctly" +fi + +if [ $((max_writes - min_writes)) -gt 1 ]; then + log_fail "mmp writes were not evenly distributed across leaf vdevs" +fi + +log_pass "mmp writes were evenly distributed across leaf vdevs" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh new file mode 100755 index 0000000000..be387637ba --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh @@ -0,0 +1,59 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +# DESCRIPTION: +# Verify MMP behaves correctly when failing to write uberblocks. +# +# STRATEGY: +# 1. Create a mirrored pool and enable multihost +# 2. Inject a 50% failure rate when writing uberblocks to a device +# 3. Delay briefly for additional MMP writes to complete +# 4. Verify the failed uberblock writes did not prevent MMP updates +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + zinject -c all + default_cleanup_noexit + log_must mmp_clear_hostid +} + +log_assert "mmp behaves correctly when failing to write uberblocks." +log_onexit cleanup + +log_must mmp_set_hostid $HOSTID1 +default_mirror_setup_noexit $DISKS +log_must zpool set multihost=on $TESTPOOL +log_must zinject -d ${DISK[0]} -e io -T write -f 50 -L uber $TESTPOOL +clear_mmp_history +uber_count=$(count_mmp_writes $TESTPOOL 3) + +if [ $uber_count -eq 0 ]; then + log_fail "mmp writes did not occur when uberblock IO errors injected" +fi + +log_pass "mmp correctly wrote uberblocks when IO errors injected" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/multihost_history.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/multihost_history.ksh new file mode 100755 index 0000000000..e831475dbc --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/multihost_history.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# zfs_multihost_history should report both writes issued and gaps +# +# STRATEGY: +# 1. Create a 2-vdev pool with mmp enabled +# 2. Delay writes by 2*MMP_INTERVAL_DEFAULT +# 3. Check multihost_history for both issued writes, and for gaps where +# no write could be issued because all vdevs are busy +# +# During the first MMP_INTERVAL period 2 MMP writes will be issued - one to +# each vdev. At the third scheduled attempt to write, at time t0+MMP_INTERVAL, +# both vdevs will still have outstanding writes, so a skipped write entry will +# be recorded in the multihost_history. + + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + log_must zinject -c all + mmp_pool_destroy $MMP_POOL $MMP_DIR + log_must mmp_clear_hostid +} + +log_assert "zfs_multihost_history records writes and skipped writes" +log_onexit cleanup + +mmp_pool_create_simple $MMP_POOL $MMP_DIR +log_must zinject -d $MMP_DIR/vdev1 -D$((2*MMP_INTERVAL_DEFAULT)):10 $MMP_POOL +log_must zinject -d $MMP_DIR/vdev2 -D$((2*MMP_INTERVAL_DEFAULT)):10 $MMP_POOL + +mmp_writes=$(count_mmp_writes $MMP_POOL $((MMP_INTERVAL_DEFAULT/1000))) +mmp_skips=$(count_skipped_mmp_writes $MMP_POOL $((MMP_INTERVAL_DEFAULT/1000))) + +if [ $mmp_writes -lt 1 ]; then + log_fail "mmp writes entries missing when delays injected" +fi + +if [ $mmp_skips -lt 1 ]; then + log_fail "mmp skipped write entries missing when delays injected" +fi + +log_pass "zfs_multihost_history records writes and skipped writes" diff --git a/usr/src/test/zfs-tests/tests/functional/mmp/setup.ksh b/usr/src/test/zfs-tests/tests/functional/mmp/setup.ksh new file mode 100755 index 0000000000..c6f8175838 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/mmp/setup.ksh @@ -0,0 +1,38 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright 2019 Joyent, Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg + +verify_runnable "global" + +case "$(uname)" in +Linux) if [ -e $HOSTID_FILE ]; then + log_unsupported "System has existing $HOSTID_FILE file" + fi + log_must set_tunable64 zfs_multihost_history $MMP_HISTORY + ;; + +SunOS) hostid >/var/tmp/zfs_test_hostid.txt + ;; +esac + +log_pass "mmp setup pass" |