diff options
Diffstat (limited to 'usr/src/test/zfs-tests/tests/functional/resilver')
7 files changed, 637 insertions, 0 deletions
diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/Makefile b/usr/src/test/zfs-tests/tests/functional/resilver/Makefile new file mode 100644 index 0000000000..dd564d19c2 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/Makefile @@ -0,0 +1,74 @@ +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2020 Joyent, Inc. +# + +include $(SRC)/Makefile.master + +PROG = sysevent + +SCRIPTS = cleanup \ + resilver_restart_001 \ + resilver_restart_002 \ + setup + +include $(SRC)/cmd/Makefile.cmd +include $(SRC)/test/Makefile.com + +ROOTOPTPKG = $(ROOT)/opt/zfs-tests +TARGETDIR = $(ROOTOPTPKG)/tests/functional/resilver + +OBJS = $(PROG:%=%.o) +SRCS = $(OBJS:%.o=%.c) +SRCFILES = resilver.cfg + +CMDS = $(PROG:%=$(TARGETDIR)/%) $(SCRIPTS:%=$(TARGETDIR)/%) +$(CMDS) := FILEMODE = 0555 + +FILES = $(SRCFILES:%=$(TARGETDIR)/%) +$(FILES) := FILEMODE = 0444 + +CPPFLAGS += -D__EXTENSIONS__ +LDLIBS += -lsysevent + +CSTD = $(CSTD_GNU99) + +all: $(PROG) + +$(PROG): $(OBJS) + $(LINK.c) $(OBJS) -o $@ $(LDFLAGS) $(LDLIBS) + $(POST_PROCESS) + +%.o: %.c + $(COMPILE.c) $< + +install: all $(CMDS) $(FILES) + +clobber: clean + -$(RM) $(PROG) + +clean: + -$(RM) $(OBJS) + +$(CMDS): $(TARGETDIR) $(PROG) + +$(FILES): $(SRCFILES) + +$(TARGETDIR): + $(INS.dir) + +$(TARGETDIR)/%: % + $(INS.file) + +$(TARGETDIR)/%: %.ksh + $(INS.rename) diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/cleanup.ksh b/usr/src/test/zfs-tests/tests/functional/resilver/cleanup.ksh new file mode 100755 index 0000000000..4dfa814245 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/cleanup.ksh @@ -0,0 +1,31 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/resilver/resilver.cfg + +verify_runnable "global" + +log_pass diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/resilver.cfg b/usr/src/test/zfs-tests/tests/functional/resilver/resilver.cfg new file mode 100644 index 0000000000..88dfd24aed --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/resilver.cfg @@ -0,0 +1,32 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4} +SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1 + +VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 )) diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh b/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh new file mode 100755 index 0000000000..6333f4197a --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh @@ -0,0 +1,196 @@ +#!/bin/ksh -p + +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/resilver/resilver.cfg + +SYSEVENT=$STF_SUITE/tests/functional/resilver/sysevent + +# +# DESCRIPTION: +# Testing resilver restart logic both with and without the deferred resilver +# feature enabled, verifying that resilver is not restarted when it is +# unecessary. +# +# STRATEGY: +# 1. Create a pool +# 2. Create four filesystems with the primary cache disable to force reads +# 3. Write four files simultaneously, one to each filesystem +# 4. Do with and without deferred resilvers enabled +# a. Replace a vdev with a spare & suspend resilver immediately +# b. Verify resilver starts properly +# c. Offline / online another vdev to introduce a new DTL range +# d. Verify resilver restart restart or defer +# e. Inject read errors on vdev that was offlined / onlned +# f. Verify that resilver did not restart +# g. Unsuspend resilver and wait for it to finish +# h. Verify that there are two resilvers and nothing is deferred +# + +function cleanup +{ + log_must set_tunable32 zfs_resilver_min_time_ms $ORIG_RESILVER_MIN_TIME + log_must set_tunable32 zfs_scan_suspend_progress \ + $ORIG_SCAN_SUSPEND_PROGRESS + log_must zinject -c all + destroy_pool $TESTPOOL + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE + [[ -n "$EVTFILE" ]] && rm -f "$EVTFILE" + [[ -n "$EVTPID" ]] && kill "$EVTPID" +} + +# count resilver events in zpool and number of deferred rsilvers on vdevs +function verify_restarts # <msg> <cnt> <defer> +{ + msg=$1 + cnt=$2 + defer=$3 + + # check the number of resilver start in events log + RESILVERS=$(wc -l $EVTFILE | awk '{ print $1 }') + log_note "expected $cnt resilver start(s)$msg, found $RESILVERS" + [[ "$RESILVERS" -ne "$cnt" ]] && + log_fail "expected $cnt resilver start(s)$msg, found $RESILVERS" + + [[ -z "$defer" ]] && return + + # use zdb to find which vdevs have the resilver defer flag + VDEV_DEFERS=$(zdb -C $TESTPOOL | awk ' + /children/ { gsub(/[^0-9]/, ""); child = $0 } + /com\.datto:resilver_defer$/ { print child } + ') + + if [[ "$defer" == "-" ]] + then + [[ -n $VDEV_DEFERS ]] && + log_fail "didn't expect any vdevs to have resilver deferred" + return + fi + + [[ $VDEV_DEFERS -eq $defer ]] || + log_fail "resilver deferred set on unexpected vdev: $VDEV_DEFERS" +} + +log_assert "Check for unnecessary resilver restarts" + +ORIG_RESILVER_MIN_TIME=$(get_tunable zfs_resilver_min_time_ms) +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable zfs_scan_suspend_progress) + +set -A RESTARTS -- '1' '2' '2' '2' +set -A VDEVS -- '' '' '' '' +set -A DEFER_RESTARTS -- '1' '1' '1' '2' +set -A DEFER_VDEVS -- '-' '2' '2' '-' + +VDEV_REPLACE="${VDEV_FILES[1]} $SPARE_VDEV_FILE" + +log_onexit cleanup + +# Monitor for resilver start events and log them to $EVTFILE as they occur +EVTFILE=$(mktemp /tmp/resilver_events.XXXXXX) +EVTPID=$($SYSEVENT -o $EVTFILE ESC_ZFS_resilver_start) +log_must test -n "$EVTPID" + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE + +log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL \ + raidz ${VDEV_FILES[@]} + +# create 4 filesystems +for fs in fs{0..3} +do + log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL/$fs +done + +# simultaneously write 16M to each of them +set -A DATAPATHS /$TESTPOOL/fs{0..3}/dat.0 +log_note "Writing data files" +for path in ${DATAPATHS[@]} +do + dd if=/dev/urandom of=$path bs=1M count=16 > /dev/null 2>&1 & +done +wait + +# test without and with deferred resilve feature enabled +for test in "without" "with" +do + log_note "Testing $test deferred resilvers" + + if [[ $test == "with" ]] + then + log_must zpool set feature@resilver_defer=enabled $TESTPOOL + RESTARTS=( "${DEFER_RESTARTS[@]}" ) + VDEVS=( "${DEFER_VDEVS[@]}" ) + VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}" + fi + + # clear the events + cp /dev/null $EVTFILE + + # limit scanning time + log_must set_tunable32 zfs_resilver_min_time_ms 50 + + # initiate a resilver and suspend the scan as soon as possible + log_must zpool replace $TESTPOOL $VDEV_REPLACE + log_must set_tunable32 zfs_scan_suspend_progress 1 + + # there should only be 1 resilver start + verify_restarts '' "${RESTARTS[0]}" "${VDEVS[0]}" + + # offline then online a vdev to introduce a new DTL range after current + # scan, which should restart (or defer) the resilver + log_must zpool offline $TESTPOOL ${VDEV_FILES[2]} + log_must zpool sync $TESTPOOL + log_must zpool online $TESTPOOL ${VDEV_FILES[2]} + log_must zpool sync $TESTPOOL + + # there should now be 2 resilver starts w/o defer, 1 with defer + verify_restarts ' after offline/online' "${RESTARTS[1]}" "${VDEVS[1]}" + + # inject read io errors on vdev and verify resilver does not restart + log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL + log_must cat ${DATAPATHS[1]} > /dev/null + log_must zinject -c all + + # there should still be 2 resilver starts w/o defer, 1 with defer + verify_restarts ' after zinject' "${RESTARTS[2]}" "${VDEVS[2]}" + + # unsuspend resilver + log_must set_tunable32 zfs_scan_suspend_progress 0 + log_must set_tunable32 zfs_resilver_min_time_ms 3000 + + # wait for resilver to finish + for iter in {0..59} + do + is_pool_resilvered $TESTPOOL && break + sleep 1 + done + is_pool_resilvered $TESTPOOL || + log_fail "resilver timed out" + + # wait for a few txg's to see if a resilver happens + log_must zpool sync $TESTPOOL + log_must zpool sync $TESTPOOL + + # there should now be 2 resilver starts + verify_restarts ' after resilver' "${RESTARTS[3]}" "${VDEVS[3]}" +done + +log_pass "Resilver did not restart unnecessarily" diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh b/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh new file mode 100644 index 0000000000..da111da9fd --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh @@ -0,0 +1,110 @@ +#!/bin/ksh -p + +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020, Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/resilver/resilver.cfg + +SYSEVENT=$STF_SUITE/tests/functional/resilver/sysevent + +# +# DESCRIPTION: +# Testing resilver completes when scan errors are encountered, but relevant +# DTL's have not been lost. +# +# STRATEGY: +# 1. Create a pool (1k recordsize) +# 2. Create a 32m file (32k records) +# 3. Inject an error halfway through the file +# 4. Start a resilver, ensure the error is triggered and that the resilver +# does not restart after finishing +# +# NB: use legacy scanning to ensure scan of specific block causes error +# + +function cleanup +{ + log_must zinject -c all + destroy_pool $TESTPOOL + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE + log_must set_tunable32 zfs_scan_legacy $ORIG_SCAN_LEGACY + [[ -n "$EVTPID" ]] && kill "$EVTPID" + [[ -n "$EVTFILE" ]] && rm -f "$EVTFILE" +} + +log_assert "Check for resilver restarts caused by scan errors" + +ORIG_SCAN_LEGACY=$(get_tunable zfs_scan_legacy) + +log_onexit cleanup + +# use legacy scan to ensure injected error will be triggered +log_must set_tunable32 zfs_scan_legacy 1 + + # create the pool and a 32M file (32k blocks) +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[0]} $SPARE_VDEV_FILE +log_must zpool create -f -O recordsize=1k $TESTPOOL ${VDEV_FILES[0]} +log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=32 > /dev/null 2>&1 + +# determine objset/object +objset=$(zdb -d $TESTPOOL/ | sed -ne 's/.*ID \([0-9]*\).*/\1/p') +object=$(ls -i /$TESTPOOL/file | awk '{print $1}') + +# inject event to cause error during resilver +log_must zinject -b `printf "%x:%x:0:3fff" $objset $object` $TESTPOOL + + +EVTFILE=$(mktemp /tmp/resilver_events.XXXXXX) +EVTPID=$($SYSEVENT -o $EVTFILE ESC_ZFS_resilver_start ESC_ZFS_resilver_finish) +log_must test -n "$EVTPID" + +# start resilver +log_must zpool attach $TESTPOOL ${VDEV_FILES[0]} $SPARE_VDEV_FILE + +log_note "waiting for read errors to start showing up" +for iter in {0..59} +do + zpool sync $TESTPOOL + err=$(zpool status $TESTPOOL | grep ${VDEV_FILES[0]} | awk '{print $3}') + (( $err > 0 )) && break + sleep 1 +done + +(( $err == 0 )) && log_fail "Unable to induce errors in resilver" + +log_note "waiting for resilver to finish" +for iter in {0..59} +do + finish=$(grep "ESC_ZFS_resilver_finish" $EVTFILE | wc -l) + (( $finish > 0 )) && break + sleep 1 +done + +(( $finish == 0 )) && log_fail "resilver took too long to finish" + +# wait a few syncs to ensure that zfs does not restart the resilver +log_must zpool sync $TESTPOOL +log_must zpool sync $TESTPOOL + +# check if resilver was restarted +start=$(grep "ESC_ZFS_resilver_start" $EVTFILE | wc -l) +(( $start != 1 )) && log_fail "resilver restarted unnecessarily" + +log_pass "Resilver did not restart unnecessarily from scan errors" diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/setup.ksh b/usr/src/test/zfs-tests/tests/functional/resilver/setup.ksh new file mode 100755 index 0000000000..4dfa814245 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/setup.ksh @@ -0,0 +1,31 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/resilver/resilver.cfg + +verify_runnable "global" + +log_pass diff --git a/usr/src/test/zfs-tests/tests/functional/resilver/sysevent.c b/usr/src/test/zfs-tests/tests/functional/resilver/sysevent.c new file mode 100644 index 0000000000..62a8b2faa2 --- /dev/null +++ b/usr/src/test/zfs-tests/tests/functional/resilver/sysevent.c @@ -0,0 +1,163 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at http://smartos.org/CDDL + * + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file. + * + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Copyright 2020 Joyent, Inc. + * + */ + +#include <err.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/debug.h> +#include <sys/sysmacros.h> +#include <sys/types.h> +#include <libsysevent.h> +#include <sys/sysevent/eventdefs.h> + +FILE *out; + +static void +process_event(sysevent_t *ev) +{ + char *class = NULL; + char *subclass = NULL; + + /* get sysevent metadata and add to the nvlist */ + class = sysevent_get_class_name(ev); + subclass = sysevent_get_subclass_name(ev); + + if (class == NULL || subclass == NULL) + errx(EXIT_FAILURE, "failed to retrieve sysevent metadata"); + + VERIFY0(strcmp(class, EC_ZFS)); + + flockfile(out); + (void) fprintf(out, "Received %s.%s event\n", class, subclass); + (void) fflush(out); + funlockfile(out); +} + +static void +child_fatal(int fd, const char *msg, ...) +{ + va_list ap; + int fail = EXIT_FAILURE; + + va_start(ap, msg); + (void) vfprintf(stderr, msg, ap); + va_end(ap); + (void) fputc('\n', stderr); + + (void) write(fd, &fail, sizeof (fail)); + (void) close(fd); + exit(EXIT_FAILURE); +} + +static void +do_child(int fd, char * const subclasses[], size_t n) +{ + sysevent_handle_t *handle; + int ret = 0; + + if ((handle = sysevent_bind_handle(process_event)) == NULL) { + child_fatal(fd, "sysevent_bind_handle() failed: %s", + strerror(errno)); + } + + if (sysevent_subscribe_event(handle, EC_ZFS, + (const char **)subclasses, n) != 0) { + child_fatal(fd, "failed to subscribe to sysevents: %s", + strerror(errno)); + } + + (void) write(fd, &ret, sizeof (ret)); + (void) close(fd); + + /* leave stderr open so any errors get captured by test harness */ + (void) fclose(stdin); + (void) fclose(stdout); + + for (;;) + (void) pause(); +} + +static void +usage(const char *name) +{ + (void) fprintf(stderr, "Usage: %s [-o outfile] zfs_event...\n", name); + exit(2); +} + +int +main(int argc, char * const argv[]) +{ + const char *outfile = NULL; + pid_t child; + int fds[2]; + int ret = 0; + int c; + + while ((c = getopt(argc, argv, "o:")) != -1) { + switch (c) { + case 'o': + outfile = optarg; + break; + case '?': + (void) fprintf(stderr, "Invalid option -%c\n", optopt); + usage(argv[0]); + } + } + + if (outfile != NULL) { + if ((out = fopen(optarg, "w")) == NULL) + err(EXIT_FAILURE, "unable to open %s", optarg); + } else { + out = stdout; + } + + VERIFY0(pipe(fds)); + + switch (child = fork()) { + case -1: + err(EXIT_FAILURE, "unable to fork"); + case 0: + do_child(fds[1], argv + optind, (size_t)(argc - optind)); + break; + default: + break; + } + + (void) close(fds[1]); + + if (read(fds[0], &ret, sizeof (ret)) < 0) + err(EXIT_FAILURE, "failure waiting on child"); + + if (ret != 0) + return (ret); + + (void) close(fds[0]); + (void) printf("%d\n", child); + return (0); +} |