9076 ZFS Performance test concurrency should be lowered for better latency

Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>

6 files changed, 56 insertions, 10 deletions

diff --git a/usr/src/test/zfs-tests/include/commands.cfg b/usr/src/test/zfs-tests/include/commands.cfg
index 1747f03da1..b85717523a 100644
--- a/usr/src/test/zfs-tests/include/commands.cfg
+++ b/usr/src/test/zfs-tests/include/commands.cfg
@@ -62,6 +62,7 @@ export USR_BIN_FILES='awk
     isainfo
     kill
     ksh
+    kstat
     ln
     logname
     ls
diff --git a/usr/src/test/zfs-tests/tests/perf/regression/random_reads.ksh b/usr/src/test/zfs-tests/tests/perf/regression/random_reads.ksh
index 655366e00c..72e6746851 100644
--- a/usr/src/test/zfs-tests/tests/perf/regression/random_reads.ksh
+++ b/usr/src/test/zfs-tests/tests/perf/regression/random_reads.ksh
@@ -25,6 +25,15 @@
 # for all fio runs. The ARC is cleared with `zinject -a` prior to each run
 # so reads will go to disk.
 #
+# Thread/Concurrency settings:
+#    PERF_NTHREADS defines the number of files created in the test filesystem,
+#    as well as the number of threads that will simultaneously drive IO to
+#    those files.  The settings chosen are from measurements in the
+#    PerfAutoESX/ZFSPerfESX Environments, selected at concurrency levels that
+#    are at peak throughput but lowest latency.  Higher concurrency introduces
+#    queue time latency and would reduce the impact of code-induced performance
+#    regressions.
+#
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/perf/perf.shlib
@@ -48,13 +57,13 @@ export TOTAL_SIZE=$(($(get_prop avail $TESTFS) * 3 / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 fi
diff --git a/usr/src/test/zfs-tests/tests/perf/regression/random_readwrite.ksh b/usr/src/test/zfs-tests/tests/perf/regression/random_readwrite.ksh
index f41a2b526e..2e02929b99 100644
--- a/usr/src/test/zfs-tests/tests/perf/regression/random_readwrite.ksh
+++ b/usr/src/test/zfs-tests/tests/perf/regression/random_readwrite.ksh
@@ -25,6 +25,15 @@
 # and used for all fio runs. The ARC is cleared with `zinject -a` prior to
 # each run so reads will go to disk.
 #
+# Thread/Concurrency settings:
+#    PERF_NTHREADS defines the number of files created in the test filesystem,
+#    as well as the number of threads that will simultaneously drive IO to
+#    those files.  The settings chosen are from measurements in the
+#    PerfAutoESX/ZFSPerfESX Environments, selected at concurrency levels that
+#    are at peak throughput but lowest latency.  Higher concurrency introduces
+#    queue time latency and would reduce the impact of code-induced performance
+#    regressions.
+#
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/perf/perf.shlib
@@ -48,13 +57,13 @@ export TOTAL_SIZE=$(($(get_prop avail $TESTFS) * 3 / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'4 8 16 64'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
 	export PERF_IOSIZES=''		# bssplit used instead
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'32 64'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=''		# bssplit used instead
 fi
diff --git a/usr/src/test/zfs-tests/tests/perf/regression/random_writes.ksh b/usr/src/test/zfs-tests/tests/perf/regression/random_writes.ksh
index 9e201a827c..d4508ef882 100644
--- a/usr/src/test/zfs-tests/tests/perf/regression/random_writes.ksh
+++ b/usr/src/test/zfs-tests/tests/perf/regression/random_writes.ksh
@@ -24,6 +24,15 @@
 # Prior to each fio run the dataset is recreated, and fio writes new files
 # into an otherwise empty pool.
 #
+# Thread/Concurrency settings:
+#    PERF_NTHREADS defines the number of files created in the test filesystem,
+#    as well as the number of threads that will simultaneously drive IO to
+#    those files.  The settings chosen are from measurements in the
+#    PerfAutoESX/ZFSPerfESX Environments, selected at concurrency levels that
+#    are at peak throughput but lowest latency.  Higher concurrency introduces
+#    queue time latency and would reduce the impact of code-induced performance
+#    regressions.
+#
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/perf/perf.shlib
@@ -47,13 +56,13 @@ export TOTAL_SIZE=$(($(get_prop avail $TESTFS) * 3 / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'32 128'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 fi
diff --git a/usr/src/test/zfs-tests/tests/perf/regression/sequential_reads.ksh b/usr/src/test/zfs-tests/tests/perf/regression/sequential_reads.ksh
index 580f2d94e4..232785648f 100644
--- a/usr/src/test/zfs-tests/tests/perf/regression/sequential_reads.ksh
+++ b/usr/src/test/zfs-tests/tests/perf/regression/sequential_reads.ksh
@@ -25,6 +25,15 @@
 # for all fio runs. The ARC is cleared with `zinject -a` prior to each run
 # so reads will go to disk.
 #
+# Thread/Concurrency settings:
+#    PERF_NTHREADS defines the number of files created in the test filesystem,
+#    as well as the number of threads that will simultaneously drive IO to
+#    those files.  The settings chosen are from measurements in the
+#    PerfAutoESX/ZFSPerfESX Environments, selected at concurrency levels that
+#    are at peak throughput but lowest latency.  Higher concurrency introduces
+#    queue time latency and would reduce the impact of code-induced performance
+#    regressions.
+#
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/perf/perf.shlib
@@ -48,13 +57,13 @@ export TOTAL_SIZE=$(($(get_prop avail $TESTFS) * 3 / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'16 64'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16 32 64'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'64k 128k 1m'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'128k 1m'}
 fi
diff --git a/usr/src/test/zfs-tests/tests/perf/regression/sequential_writes.ksh b/usr/src/test/zfs-tests/tests/perf/regression/sequential_writes.ksh
index 493a3d18b7..2799c43336 100644
--- a/usr/src/test/zfs-tests/tests/perf/regression/sequential_writes.ksh
+++ b/usr/src/test/zfs-tests/tests/perf/regression/sequential_writes.ksh
@@ -24,6 +24,15 @@
 # Prior to each fio run the dataset is recreated, and fio writes new files
 # into an otherwise empty pool.
 #
+# Thread/Concurrency settings:
+#    PERF_NTHREADS defines the number of files created in the test filesystem,
+#    as well as the number of threads that will simultaneously drive IO to
+#    those files.  The settings chosen are from measurements in the
+#    PerfAutoESX/ZFSPerfESX Environments, selected at concurrency levels that
+#    are at peak throughput but lowest latency.  Higher concurrency introduces
+#    queue time latency and would reduce the impact of code-induced performance
+#    regressions.
+#
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/perf/perf.shlib
@@ -47,13 +56,13 @@ export TOTAL_SIZE=$(($(get_prop avail $TESTFS) * 3 / 2))
 if [[ -n $PERF_REGRESSION_WEEKLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_WEEKLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'weekly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'8 16'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'1 4 8 16 32 64 128'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'0 1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'}
 elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
 	export PERF_RUNTIME=${PERF_RUNTIME:-$PERF_RUNTIME_NIGHTLY}
 	export PERF_RUNTYPE=${PERF_RUNTYPE:-'nightly'}
-	export PERF_NTHREADS=${PERF_NTHREADS:-'64 128'}
+	export PERF_NTHREADS=${PERF_NTHREADS:-'16 32'}
 	export PERF_SYNC_TYPES=${PERF_SYNC_TYPES:-'1'}
 	export PERF_IOSIZES=${PERF_IOSIZES:-'8k 128k 1m'}
 fi