6641803 fmd deadlock in fmd_case_hash_lookup and fmd_case_rele

6645597 fmd crashed in fmd_event_match
6664741 fmd core dumping on Doradi with build snv_82
6667072 fmd crashed at fmd_api_transport_impl

5 files changed, 101 insertions, 37 deletions

diff --git a/usr/src/cmd/fm/fmd/common/fmd.c b/usr/src/cmd/fm/fmd/common/fmd.c
index c3569de11e..c4c5a0b11c 100644
--- a/usr/src/cmd/fm/fmd/common/fmd.c
+++ b/usr/src/cmd/fm/fmd/common/fmd.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -783,7 +783,6 @@ fmd_run(fmd_t *dp, int pfd)
 	fmd_xprt_suspend_all();
 
 	(void) door_server_create(fmd_door);
-	fmd_dr_init();
 
 	dp->d_rmod->mod_timerids = fmd_idspace_create(dp->d_pname, 1, 16);
 	dp->d_timers = fmd_timerq_create();
@@ -902,6 +901,11 @@ fmd_run(fmd_t *dp, int pfd)
 	fmd_modhash_loadall(dp->d_mod_hash, pap, &fmd_proc_ops, NULL);
 
 	/*
+	 * Subscribe to sysevents after all modules have been loaded.
+	 */
+	fmd_dr_init();
+
+	/*
 	 * With all modules loaded, replay fault events from the ASRU cache for
 	 * any ASRUs that must be retired, replay error events from the errlog
 	 * that did not finish processing the last time ran, and then release
diff --git a/usr/src/cmd/fm/fmd/common/fmd_case.c b/usr/src/cmd/fm/fmd/common/fmd_case.c
index 9af5eaf73c..bc873bf246 100644
--- a/usr/src/cmd/fm/fmd/common/fmd_case.c
+++ b/usr/src/cmd/fm/fmd/common/fmd_case.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -133,6 +133,8 @@ static const char *const _fmd_case_snames[] = {
 
 extern volatile uint32_t fmd_asru_fake_not_present;
 
+static fmd_case_impl_t *fmd_case_tryhold(fmd_case_impl_t *);
+
 fmd_case_hash_t *
 fmd_case_hash_create(void)
 {
@@ -440,9 +442,13 @@ fmd_case_hash_lookup(fmd_case_hash_t *chp, const char *uuid)
 			break;
 	}
 
+	/*
+	 * If deleting bit is set, treat the case as if it doesn't exist.
+	 */
 	if (cip != NULL)
-		fmd_case_hold((fmd_case_t *)cip);
-	else
+		cip = fmd_case_tryhold(cip);
+
+	if (cip == NULL)
 		(void) fmd_set_errno(EFMD_CASE_INVAL);
 
 	(void) pthread_rwlock_unlock(&chp->ch_lock);
@@ -459,8 +465,8 @@ fmd_case_hash_insert(fmd_case_hash_t *chp, fmd_case_impl_t *cip)
 	h = fmd_strhash(cip->ci_uuid) % chp->ch_hashlen;
 
 	for (eip = chp->ch_hash[h]; eip != NULL; eip = eip->ci_next) {
-		if (strcmp(cip->ci_uuid, eip->ci_uuid) == 0) {
-			fmd_case_hold((fmd_case_t *)eip);
+		if (strcmp(cip->ci_uuid, eip->ci_uuid) == 0 &&
+		    fmd_case_tryhold(eip) != NULL) {
 			(void) pthread_rwlock_unlock(&chp->ch_lock);
 			return (eip); /* uuid already present */
 		}
@@ -482,6 +488,11 @@ fmd_case_hash_delete(fmd_case_hash_t *chp, fmd_case_impl_t *cip)
 	fmd_case_impl_t *cp, **pp;
 	uint_t h;
 
+	ASSERT(MUTEX_HELD(&cip->ci_lock));
+
+	cip->ci_flags |= FMD_CF_DELETING;
+	(void) pthread_mutex_unlock(&cip->ci_lock);
+
 	(void) pthread_rwlock_wrlock(&chp->ch_lock);
 
 	h = fmd_strhash(cip->ci_uuid) % chp->ch_hashlen;
@@ -506,6 +517,9 @@ fmd_case_hash_delete(fmd_case_hash_t *chp, fmd_case_impl_t *cip)
 	chp->ch_count--;
 
 	(void) pthread_rwlock_unlock(&chp->ch_lock);
+
+	(void) pthread_mutex_lock(&cip->ci_lock);
+	ASSERT(cip->ci_flags & FMD_CF_DELETING);
 }
 
 fmd_case_t *
@@ -701,8 +715,7 @@ fmd_case_hold(fmd_case_t *cp)
 	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;
 
 	(void) pthread_mutex_lock(&cip->ci_lock);
-	cip->ci_refs++;
-	ASSERT(cip->ci_refs != 0);
+	fmd_case_hold_locked(cp);
 	(void) pthread_mutex_unlock(&cip->ci_lock);
 }
 
@@ -712,10 +725,31 @@ fmd_case_hold_locked(fmd_case_t *cp)
 	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;
 
 	ASSERT(MUTEX_HELD(&cip->ci_lock));
+	if (cip->ci_flags & FMD_CF_DELETING)
+		fmd_panic("attempt to hold a deleting case %p (%s)\n",
+		    (void *)cip, cip->ci_uuid);
 	cip->ci_refs++;
 	ASSERT(cip->ci_refs != 0);
 }
 
+static fmd_case_impl_t *
+fmd_case_tryhold(fmd_case_impl_t *cip)
+{
+	/*
+	 * If the case's "deleting" bit is unset, hold and return case,
+	 * otherwise, return NULL.
+	 */
+	(void) pthread_mutex_lock(&cip->ci_lock);
+	if (cip->ci_flags & FMD_CF_DELETING) {
+		(void) pthread_mutex_unlock(&cip->ci_lock);
+		cip = NULL;
+	} else {
+		fmd_case_hold_locked((fmd_case_t *)cip);
+		(void) pthread_mutex_unlock(&cip->ci_lock);
+	}
+	return (cip);
+}
+
 void
 fmd_case_rele(fmd_case_t *cp)
 {
diff --git a/usr/src/cmd/fm/fmd/common/fmd_case.h b/usr/src/cmd/fm/fmd/common/fmd_case.h
index 6724c543ed..1cae35769c 100644
--- a/usr/src/cmd/fm/fmd/common/fmd_case.h
+++ b/usr/src/cmd/fm/fmd/common/fmd_case.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -91,6 +91,7 @@ typedef struct fmd_case_impl {
 #define	FMD_CF_REPAIRED		0x08	/* case has been repaired */
 #define	FMD_CF_REPAIRING	0x10	/* case repair in progress */
 #define	FMD_CF_INVISIBLE	0x20	/* case should be invisible */
+#define	FMD_CF_DELETING		0x40	/* case is about to be deleted */
 
 typedef struct fmd_case_hash {
 	pthread_rwlock_t ch_lock;	/* lock protecting case hash */
diff --git a/usr/src/cmd/fm/fmd/common/fmd_event.c b/usr/src/cmd/fm/fmd/common/fmd_event.c
index eddd1e2b7a..b0908f0c00 100644
--- a/usr/src/cmd/fm/fmd/common/fmd_event.c
+++ b/usr/src/cmd/fm/fmd/common/fmd_event.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -345,12 +345,15 @@ fmd_event_match(fmd_event_t *e, uint_t type, const void *data)
 {
 	fmd_event_impl_t *ep = (fmd_event_impl_t *)e;
 
+	if (ep->ev_type != type)
+		return (0);
+
 	if (type == FMD_EVT_PROTOCOL)
-		return (ep->ev_type == type && fmd_strmatch(ep->ev_data, data));
+		return (fmd_strmatch(ep->ev_data, data));
 	else if (type == FMD_EVT_TIMEOUT)
 		return ((id_t)data == ((fmd_modtimer_t *)ep->ev_data)->mt_id);
 	else
-		return (ep->ev_type == type && ep->ev_data == data);
+		return (ep->ev_data == data);
 }
 
 int
diff --git a/usr/src/cmd/fm/fmd/common/fmd_sysevent.c b/usr/src/cmd/fm/fmd/common/fmd_sysevent.c
index eb42ea2528..0172dcbe8f 100644
--- a/usr/src/cmd/fm/fmd/common/fmd_sysevent.c
+++ b/usr/src/cmd/fm/fmd/common/fmd_sysevent.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -65,6 +65,7 @@ static char *sysev_sid;		/* event channel subscriber identifier */
 static void *sysev_evc;		/* event channel cookie from evc_bind */
 
 static fmd_xprt_t *sysev_xprt;
+static int sysev_xprt_refcnt;
 static fmd_hdl_t *sysev_hdl;
 
 static struct sysev_stats {
@@ -81,9 +82,11 @@ static struct sysev_stats {
 	{ "eagain", FMD_TYPE_UINT64, "events retried due to low memory" },
 };
 
-static pthread_cond_t sysev_replay_cv = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t sysev_replay_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t sysev_cv = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t sysev_mutex = PTHREAD_MUTEX_INITIALIZER;
 static int sysev_replay_wait = 1;
+static int sysev_exiting;
+
 
 /*
  * Receive an event from the SysEvent channel and post it to our transport.
@@ -100,36 +103,46 @@ sysev_recv(sysevent_t *sep, void *arg)
 	fmd_xprt_t *xp = arg;
 	nvlist_t *nvl;
 	hrtime_t hrt;
-
-	(void) pthread_mutex_lock(&sysev_replay_mutex);
+	int rc = 0;
+
+	(void) pthread_mutex_lock(&sysev_mutex);
+	if (sysev_exiting == 1) {
+		while (sysev_xprt_refcnt > 0)
+			(void) pthread_cond_wait(&sysev_cv, &sysev_mutex);
+		(void) pthread_mutex_unlock(&sysev_mutex);
+		return (EAGAIN);
+	}
+	sysev_xprt_refcnt++;
 	while (sysev_replay_wait)
-		(void) pthread_cond_wait(&sysev_replay_cv, &sysev_replay_mutex);
-	(void) pthread_mutex_unlock(&sysev_replay_mutex);
+		(void) pthread_cond_wait(&sysev_cv, &sysev_mutex);
+	(void) pthread_mutex_unlock(&sysev_mutex);
 
 	if (strcmp(sysevent_get_class_name(sep), EC_FM) != 0) {
 		fmd_hdl_error(sysev_hdl, "discarding event 0x%llx: unexpected"
 		    " transport class %s\n", seq, sysevent_get_class_name(sep));
 		sysev_stats.bad_class.fmds_value.ui64++;
-		return (0);
-	}
-
-	if (sysevent_get_attr_list(sep, &nvl) != 0) {
+	} else if (sysevent_get_attr_list(sep, &nvl) != 0) {
 		if (errno == EAGAIN || errno == ENOMEM) {
 			fmd_modhash_tryapply(fmd.d_mod_hash, fmd_module_trygc);
 			fmd_scheme_hash_trygc(fmd.d_schemes);
 			sysev_stats.eagain.fmds_value.ui64++;
-			return (EAGAIN);
+			rc = EAGAIN;
+		} else {
+			fmd_hdl_error(sysev_hdl, "discarding event 0x%llx: "
+			    "missing or invalid payload", seq);
+			sysev_stats.bad_attr.fmds_value.ui64++;
 		}
-
-		fmd_hdl_error(sysev_hdl, "discarding event 0x%llx: missing "
-		    "or invalid payload", seq);
-		sysev_stats.bad_attr.fmds_value.ui64++;
-		return (0);
+	} else {
+		sysevent_get_time(sep, &hrt);
+		fmd_xprt_post(sysev_hdl, xp, nvl, hrt);
 	}
 
-	sysevent_get_time(sep, &hrt);
-	fmd_xprt_post(sysev_hdl, xp, nvl, hrt);
-	return (0);
+	(void) pthread_mutex_lock(&sysev_mutex);
+	if (--sysev_xprt_refcnt == 0 && sysev_exiting == 1)
+		(void) pthread_cond_broadcast(&sysev_cv);
+	(void) pthread_mutex_unlock(&sysev_mutex);
+
+	return (rc);
 }
 
 /*
@@ -341,10 +354,10 @@ next:
 
 	(void) close(fd);
 done:
-	(void) pthread_mutex_lock(&sysev_replay_mutex);
+	(void) pthread_mutex_lock(&sysev_mutex);
 	sysev_replay_wait = 0;
-	(void) pthread_cond_broadcast(&sysev_replay_cv);
-	(void) pthread_mutex_unlock(&sysev_replay_mutex);
+	(void) pthread_cond_broadcast(&sysev_cv);
+	(void) pthread_mutex_unlock(&sysev_mutex);
 }
 
 static const fmd_prop_t sysev_props[] = {
@@ -453,8 +466,17 @@ sysev_fini(fmd_hdl_t *hdl)
 		sysevent_evc_unbind(sysev_evc);
 	}
 
-	if (sysev_xprt != NULL)
+	if (sysev_xprt != NULL) {
+		/*
+		 * Wait callback returns before destroy the transport.
+		 */
+		(void) pthread_mutex_lock(&sysev_mutex);
+		sysev_exiting = 1;
+		while (sysev_xprt_refcnt > 0)
+			(void) pthread_cond_wait(&sysev_cv, &sysev_mutex);
+		(void) pthread_mutex_unlock(&sysev_mutex);
 		fmd_xprt_close(hdl, sysev_xprt);
+	}
 
 	fmd_prop_free_string(hdl, sysev_class);
 	fmd_prop_free_string(hdl, sysev_channel);