14095 Dtrace probes for nbmand conflict identification

Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Joyce McIntosh <joyce.mcintosh@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Approved by: Robert Mustacchi <rm@fingolfin.org>

5 files changed, 79 insertions, 4 deletions

diff --git a/usr/src/cmd/smbsrv/dtrace/Makefile b/usr/src/cmd/smbsrv/dtrace/Makefile
index 0ed2c858af..cf25f9c735 100644
--- a/usr/src/cmd/smbsrv/dtrace/Makefile
+++ b/usr/src/cmd/smbsrv/dtrace/Makefile
@@ -26,7 +26,8 @@
 #
 
 SRCS=	smbd-all.d smbd-authsvc.d smbd-doorsvc.d smbd-pipesvc.d \
-	smbnode.d smbsrv.d smbvfs.d smb-trace.d smb2-trace.d
+	smbnode.d smbsrv.d smbvfs.d smb-trace.d smb2-trace.d \
+	nbl-conflict.d
 
 include ../../Makefile.cmd
 
diff --git a/usr/src/cmd/smbsrv/dtrace/nbl-conflict.d b/usr/src/cmd/smbsrv/dtrace/nbl-conflict.d
new file mode 100644
index 0000000000..da66f76679
--- /dev/null
+++ b/usr/src/cmd/smbsrv/dtrace/nbl-conflict.d
@@ -0,0 +1,56 @@
+#!/usr/sbin/dtrace -s
+
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2019 Nexenta Systems, Inc.  All rights reserved.
+ */
+
+/*
+ * This dtrace script shows how to track down the owners of locks
+ * that prevent I/O in filesystems with mandatory locking enabled
+ * (eg. ZFS with nbmand=on).  This script is not in any way specific
+ * to SMB, but this problem is most often seen when SMB is in use
+ * because SMB requires mandatory locking semantics.
+ *
+ * Run this script, eg. dtrace -s nbl-conflict.d
+ * taking note of these fields in the dtrace output:
+ *	conflict_lock:  .l_sysid .l_pid
+ *	conflict_shrlock: .s_sysid .s_pid
+ *
+ * The sysid values tell you if a local or remote owner has the
+ * lock or share preventing I/O, and the pid tells which process.
+ */
+
+sdt::nbl_lock_conflict:conflict_lock
+{
+	this->lock = (lock_descriptor_t *)arg0;
+	print(this->lock->l_flock);
+}
+
+sdt::nbl_share_conflict:conflict_shrlock
+{
+	this->shrl = (struct shrlock *)arg0;
+	print(*(this->shrl));
+}
+
+/*
+ * The above probe action in nbl_share_conflict shows conflicts
+ * with read/write operations (eg. from the NFS server).
+ * This probe action shows share reservation conflicts at open.
+ * (Remove this if you're only interested in I/O conflicts.)
+ */
+sdt::add_share:conflict_shrlock
+{
+	this->shrl = (struct shrlock *)arg0;
+	print(*(this->shrl));
+}
diff --git a/usr/src/pkg/manifests/service-file-system-smb.p5m b/usr/src/pkg/manifests/service-file-system-smb.p5m
index b767a030fe..e87ab1d009 100644
--- a/usr/src/pkg/manifests/service-file-system-smb.p5m
+++ b/usr/src/pkg/manifests/service-file-system-smb.p5m
@@ -67,6 +67,7 @@ link path=usr/lib/security/pam_smb_passwd.so target=pam_smb_passwd.so.1
 file path=usr/lib/security/pam_smb_passwd.so.1
 dir  path=usr/lib/smbsrv
 dir  path=usr/lib/smbsrv/dtrace
+file path=usr/lib/smbsrv/dtrace/nbl-conflict.d mode=0555
 file path=usr/lib/smbsrv/dtrace/smb-trace.d mode=0555
 file path=usr/lib/smbsrv/dtrace/smb2-trace.d mode=0555
 file path=usr/lib/smbsrv/dtrace/smbd-all.d mode=0555
diff --git a/usr/src/uts/common/os/flock.c b/usr/src/uts/common/os/flock.c
index 78907db25c..77ec4fe9ce 100644
--- a/usr/src/uts/common/os/flock.c
+++ b/usr/src/uts/common/os/flock.c
@@ -28,7 +28,7 @@
 /*	All Rights Reserved */
 
 /*
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2019 Nexenta by DDN, Inc. All rights reserved.
  * Copyright 2015 Joyent, Inc.
  */
 
@@ -4318,6 +4318,8 @@ nbl_lock_conflict(vnode_t *vp, nbl_op_t op, u_offset_t offset,
 		    lock->l_flock.l_pid != pid) &&
 		    lock_blocks_io(op, offset, length,
 		    lock->l_type, lock->l_start, lock->l_end)) {
+			DTRACE_PROBE1(conflict_lock,
+			    lock_descriptor_t *, lock);
 			conflict = 1;
 			break;
 		}
diff --git a/usr/src/uts/common/os/share.c b/usr/src/uts/common/os/share.c
index 55a7422868..6a06be2d9c 100644
--- a/usr/src/uts/common/os/share.c
+++ b/usr/src/uts/common/os/share.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2019 Nexenta by DDN, Inc. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -125,6 +125,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 				    (shr->s_deny & F_RDDNY) ||
 				    (shrl->shr->s_access & F_WRACC)) {
 					mutex_exit(&vp->v_lock);
+					DTRACE_PROBE1(conflict_shrlock,
+					    struct shrlock *, shrl->shr);
 					return (EAGAIN);
 				}
 				/*
@@ -135,6 +137,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 				if (isreadonly(vp))
 					break;
 				mutex_exit(&vp->v_lock);
+				DTRACE_PROBE1(conflict_shrlock,
+				    struct shrlock *, shrl->shr);
 				return (EAGAIN);
 			}
 			/*
@@ -147,6 +151,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 			    (shrl->shr->s_access == F_RDACC))
 				break;
 			mutex_exit(&vp->v_lock);
+			DTRACE_PROBE1(conflict_shrlock,
+			    struct shrlock *, shrl->shr);
 			return (EAGAIN);
 		}
 
@@ -171,6 +177,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 			    (shrl->shr->s_deny & F_RDDNY) ||
 			    (shrl->shr->s_access & F_WRACC)) {
 				mutex_exit(&vp->v_lock);
+				DTRACE_PROBE1(conflict_shrlock,
+				    struct shrlock *, shrl->shr);
 				return (EAGAIN);
 			}
 			/*
@@ -183,6 +191,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 					break;
 				}
 				mutex_exit(&vp->v_lock);
+				DTRACE_PROBE1(conflict_shrlock,
+				    struct shrlock *, shrl->shr);
 				return (EAGAIN);
 			}
 			/*
@@ -199,6 +209,8 @@ add_share(struct vnode *vp, struct shrlock *shr)
 		if ((shr->s_access & shrl->shr->s_deny) ||
 		    (shr->s_deny & shrl->shr->s_access)) {
 			mutex_exit(&vp->v_lock);
+			DTRACE_PROBE1(conflict_shrlock,
+			    struct shrlock *, shrl->shr);
 			return (EAGAIN);
 		}
 	}
@@ -609,8 +621,11 @@ nbl_share_conflict(vnode_t *vp, nbl_op_t op, caller_context_t *ct)
 			break;
 #endif
 		}
-		if (conflict)
+		if (conflict) {
+			DTRACE_PROBE1(conflict_shrlock,
+			    struct shrlock *, shrl->shr);
 			break;
+		}
 	}
 
 	mutex_exit(&vp->v_lock);