6546122 ip_thread_exit() ill/ipif/nce refcount checks missing post-IP-instances

6 files changed, 75 insertions, 4 deletions

diff --git a/usr/src/uts/common/disp/thread.c b/usr/src/uts/common/disp/thread.c
index b4a34c9822..2335734849 100644
--- a/usr/src/uts/common/disp/thread.c
+++ b/usr/src/uts/common/disp/thread.c
@@ -75,6 +75,8 @@
 #include <sys/kdi.h>
 #include <sys/waitq.h>
 #include <sys/cpucaps.h>
+#include <inet/ip.h>
+#include <inet/ip_if.h>
 
 struct kmem_cache *thread_cache;	/* cache of free threads */
 struct kmem_cache *lwp_cache;		/* cache of free lwps */
@@ -559,8 +561,11 @@ thread_rele(kthread_t *t)
 	}
 }
 
-
-void	(*ip_cleanup_func)(void);
+/*
+ * This is a function which is called from thread_exit
+ * that can be used to debug reference count issues in IP.
+ */
+void (*ip_cleanup_func)(void);
 
 void
 thread_exit()
diff --git a/usr/src/uts/common/inet/ip/ip.c b/usr/src/uts/common/inet/ip/ip.c
index 60c7fc404d..fb1b1990a7 100644
--- a/usr/src/uts/common/inet/ip/ip.c
+++ b/usr/src/uts/common/inet/ip/ip.c
@@ -5943,6 +5943,11 @@ ip_ddi_init(void)
 	ip_ire_g_init();
 	ip_net_g_init();
 
+#ifdef ILL_DEBUG
+	/* Default cleanup function */
+	ip_cleanup_func = ip_thread_exit;
+#endif
+
 	/*
 	 * We want to be informed each time a stack is created or
 	 * destroyed in the kernel, so we can maintain the
diff --git a/usr/src/uts/common/inet/ip/ip_if.c b/usr/src/uts/common/inet/ip/ip_if.c
index 138f733242..8ba076d0ff 100644
--- a/usr/src/uts/common/inet/ip/ip_if.c
+++ b/usr/src/uts/common/inet/ip/ip_if.c
@@ -6596,7 +6596,7 @@ ill_thread_exit(ill_t *ill, void *dummy)
 
 #ifdef ILL_DEBUG
 void
-ip_thread_exit(ip_stack_t *ipst)
+ip_thread_exit_stack(ip_stack_t *ipst)
 {
 	ill_t	*ill;
 	ipif_t	*ipif;
@@ -6619,6 +6619,23 @@ ip_thread_exit(ip_stack_t *ipst)
 }
 
 /*
+ * This is a function which is called from thread_exit
+ * that can be used to debug reference count issues in IP. See comment in
+ * <inet/ip.h> on how it is used.
+ */
+void
+ip_thread_exit(void)
+{
+	netstack_t *ns;
+
+	ns = netstack_get_current();
+	if (ns != NULL) {
+		ip_thread_exit_stack(ns->netstack_ip);
+		netstack_rele(ns);
+	}
+}
+
+/*
  * Called when ipif is unplumbed or when memory alloc fails
  */
 void
diff --git a/usr/src/uts/common/inet/ip_if.h b/usr/src/uts/common/inet/ip_if.h
index 592846bc43..eec04e3856 100644
--- a/usr/src/uts/common/inet/ip_if.h
+++ b/usr/src/uts/common/inet/ip_if.h
@@ -473,6 +473,48 @@ extern int ip_sioctl_move(ipif_t *, sin_t *, queue_t *, mblk_t *,
 
 extern	void	conn_delete_ire(conn_t *, caddr_t);
 
+/*
+ * This is a function which is called from thread_exit
+ * that can be used to debug reference count issues in IP.
+ *
+ * Notes on reference tracing on ill, ipif, ire, nce data structures:
+ *
+ * The current model of references on an ipif or ill is purely based on threads
+ * acquiring a reference by doing a lookup on the ill or ipif or by calling a
+ * refhold function on the ill or ipif. In particular any data structure that
+ * points to an ipif or ill does not explicitly contribute to a reference on the
+ * ill or ipif. More details may be seen in the block comment above ipif_down().
+ * Thus in the quiescent state an ill or ipif has a refcnt of zero. Similarly
+ * when a thread exits, there can't be any references on the ipif or ill due to
+ * the exiting thread.
+ *
+ * As a debugging aid, the refhold and refrele functions call into tracing
+ * functions that record the stack trace of the caller and the references
+ * acquired or released by the calling thread, hashed by the thread id. On
+ * thread exit, ipif_thread_exit and ill_thread_exit verify that there are no
+ * outstanding references to the ipif or ill from the exiting thread.
+ *
+ * In the case of ires and nces, the model is slightly different. Typically each
+ * ire pointing to an nce contributes to the nce_refcnt. Similarly a conn_t
+ * pointing to an ire also contributes to the ire_refcnt. Excluding the above
+ * special cases, the tracing behavior is similar to the tracing on ipif / ill.
+ * Traces are neither recorded nor verified in the exception cases, and the code
+ * is careful to use the right refhold and refrele functions. On thread exit
+ * ire_thread_exit, nce_thread_exit does the verification that are no
+ * outstanding references on the ire / nce from the exiting thread.
+ *
+ * The reference verification is driven from thread_exit() which calls into IP
+ * via a function pointer ip_cleanup_func into the verification function
+ * ip_thread_exit. This debugging aid may be helpful in tracing missing
+ * refrele's on a debug kernel. On a non-debug kernel, these missing refrele's
+ * are noticeable only when an interface is being unplumbed, and the unplumb
+ * hangs, long after the missing refrele. On a debug kernel, the traces
+ * (th_trace_t) which contain the stack backtraces can be examined on a crash
+ * dump to locate the missing refrele.
+ */
+extern void (*ip_cleanup_func)(void);
+extern void ip_thread_exit(void);
+
 #endif /* _KERNEL */
 
 #ifdef	__cplusplus
diff --git a/usr/src/uts/common/os/netstack.c b/usr/src/uts/common/os/netstack.c
index 60ee49f8ed..7c7f0158dc 100644
--- a/usr/src/uts/common/os/netstack.c
+++ b/usr/src/uts/common/os/netstack.c
@@ -778,7 +778,7 @@ netstack_do_destroy(void)
  * Increases the reference count, caller must do a netstack_rele.
  * It can't be called after zone_destroy() has started.
  */
-static netstack_t *
+netstack_t *
 netstack_get_current(void)
 {
 	netstack_t *ns;
diff --git a/usr/src/uts/common/sys/netstack.h b/usr/src/uts/common/sys/netstack.h
index 07ad7b5c30..df1b1ca732 100644
--- a/usr/src/uts/common/sys/netstack.h
+++ b/usr/src/uts/common/sys/netstack.h
@@ -202,6 +202,8 @@ extern netstack_t *netstack_find_by_zoneid(zoneid_t);
 extern zoneid_t netstackid_to_zoneid(netstackid_t);
 extern netstackid_t zoneid_to_netstackid(zoneid_t);
 
+extern netstack_t *netstack_get_current(void);
+
 /*
  * Register interest in changes to the set of netstacks.
  * The createfn and destroyfn are required, but the shutdownfn can be