6548350 pkcs11 pthread_atfork() covers insufficient locks

7 files changed, 269 insertions, 14 deletions

diff --git a/usr/src/lib/pkcs11/libpkcs11/common/pkcs11General.c b/usr/src/lib/pkcs11/libpkcs11/common/pkcs11General.c
index 9d1f4bbf8e..bc0d9a1239 100644
--- a/usr/src/lib/pkcs11/libpkcs11/common/pkcs11General.c
+++ b/usr/src/lib/pkcs11/libpkcs11/common/pkcs11General.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -114,6 +113,7 @@ static struct CK_FUNCTION_LIST functionList = {
 boolean_t pkcs11_initialized = B_FALSE;
 boolean_t pkcs11_cant_create_threads = B_FALSE;
 boolean_t fini_called = B_FALSE;
+static boolean_t pkcs11_atfork_initialized = B_FALSE;
 static pid_t pkcs11_pid = 0;
 
 /* protects pkcs11_[initialized|pid], and fastpath */
@@ -123,32 +123,71 @@ static CK_RV finalize_common(CK_VOID_PTR pReserved);
 static void pkcs11_fini();
 
 /*
- * Ensure that before a fork, globalmutex is taken.
+ * Ensure that before a fork, all mutexes are taken.
+ * Order:
+ * 1. globalmutex
+ * 2. slottable->st_mutex
+ * 3. all slottable->st_slots' mutexes
  */
 static void
 pkcs11_fork_prepare(void)
 {
+	int i;
 	(void) pthread_mutex_lock(&globalmutex);
+	if (slottable != NULL) {
+		(void) pthread_mutex_lock(&slottable->st_mutex);
+
+		/* Take the sl_mutex of all slots */
+		for (i = slottable->st_first; i <= slottable->st_last; i++) {
+			if (slottable->st_slots[i] != NULL) {
+				(void) pthread_mutex_lock(
+				    &slottable->st_slots[i]->sl_mutex);
+			}
+		}
+	}
 }
 
 
 /*
- * Ensure that after a fork, in the parent, globalmutex is released.
+ * Ensure that after a fork, in the parent, all mutexes are released in opposite
+ * order to pkcs11_fork_prepare().
  */
 static void
 pkcs11_fork_parent(void)
 {
+	int i;
+	if (slottable != NULL) {
+		/* Release the sl_mutex of all slots */
+		for (i = slottable->st_first; i <= slottable->st_last; i++) {
+			if (slottable->st_slots[i] != NULL) {
+				(void) pthread_mutex_unlock(
+				    &slottable->st_slots[i]->sl_mutex);
+			}
+		}
+		(void) pthread_mutex_unlock(&slottable->st_mutex);
+	}
 	(void) pthread_mutex_unlock(&globalmutex);
 }
 
 
 /*
- * Ensure that after a fork, in the child, globalmutex is released
- * and cleanup is done.
+ * Ensure that after a fork, in the child, all mutexes are released in opposite
+ * order to pkcs11_fork_prepare() and cleanup is done.
  */
 static void
 pkcs11_fork_child(void)
 {
+	int i;
+	if (slottable != NULL) {
+		/* Release the sl_mutex of all slots */
+		for (i = slottable->st_first; i <= slottable->st_last; i++) {
+			if (slottable->st_slots[i] != NULL) {
+				(void) pthread_mutex_unlock(
+				    &slottable->st_slots[i]->sl_mutex);
+			}
+		}
+		(void) pthread_mutex_unlock(&slottable->st_mutex);
+	}
 	(void) pthread_mutex_unlock(&globalmutex);
 	pkcs11_fini();
 }
@@ -256,8 +295,12 @@ C_Initialize(CK_VOID_PTR pInitArgs)
 
 	pkcs11_initialized = B_TRUE;
 	pkcs11_pid = initialize_pid;
-	(void) pthread_atfork(pkcs11_fork_prepare,
-	    pkcs11_fork_parent, pkcs11_fork_child);
+	/* Children inherit parent's atfork handlers */
+	if (!pkcs11_atfork_initialized) {
+		(void) pthread_atfork(pkcs11_fork_prepare,
+		    pkcs11_fork_parent, pkcs11_fork_child);
+		pkcs11_atfork_initialized = B_TRUE;
+	}
 	(void) pthread_mutex_unlock(&globalmutex);
 
 	/* Cleanup data structures no longer needed */
diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelGeneral.c b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelGeneral.c
index f57086c881..2024f138d9 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelGeneral.c
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelGeneral.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -110,6 +110,7 @@ static struct CK_FUNCTION_LIST functionList = {
 };
 
 boolean_t kernel_initialized = B_FALSE;
+static boolean_t kernel_atfork_initialized = B_FALSE;
 static pid_t kernel_pid = 0;
 
 int kernel_fd = -1;
@@ -125,6 +126,9 @@ kmh_elem_t **kernel_mechhash;	/* Hash table for kCF mech numbers */
 static void finalize_common();
 static void cleanup_library();
 static void kernel_fini();
+static void kernel_fork_prepare();
+static void kernel_fork_parent();
+static void kernel_fork_child();
 
 CK_RV
 C_Initialize(CK_VOID_PTR pInitArgs)
@@ -242,6 +246,13 @@ C_Initialize(CK_VOID_PTR pInitArgs)
 	kernel_initialized = B_TRUE;
 	kernel_pid = initialize_pid;
 
+	/* Children inherit parent's atfork handlers */
+	if (!kernel_atfork_initialized) {
+		(void) pthread_atfork(kernel_fork_prepare, kernel_fork_parent,
+		    kernel_fork_child);
+		kernel_atfork_initialized = B_TRUE;
+	}
+
 	(void) pthread_mutex_unlock(&globalmutex);
 
 	return (CKR_OK);
@@ -446,6 +457,50 @@ C_GetFunctionStatus(CK_SESSION_HANDLE hSession)
 }
 
 /*
+ * Take out all mutexes before fork.
+ * Order:
+ * 1. globalmutex
+ * 2. all slots mutexes (and all their sessions) via
+ *    kernel_acquire_all_slots_mutexes()
+ * 3. obj_delay_freed.obj_to_be_free_mutex;
+ * 4. ses_delay_freed.ses_to_be_free_mutex
+ */
+void
+kernel_fork_prepare()
+{
+	(void) pthread_mutex_lock(&globalmutex);
+	kernel_acquire_all_slots_mutexes();
+	(void) pthread_mutex_lock(
+	    &obj_delay_freed.obj_to_be_free_mutex);
+	(void) pthread_mutex_lock(
+	    &ses_delay_freed.ses_to_be_free_mutex);
+}
+
+/* Release in opposite order to kernel_fork_prepare(). */
+void
+kernel_fork_parent()
+{
+	(void) pthread_mutex_unlock(
+	    &ses_delay_freed.ses_to_be_free_mutex);
+	(void) pthread_mutex_unlock(
+	    &obj_delay_freed.obj_to_be_free_mutex);
+	kernel_release_all_slots_mutexes();
+	(void) pthread_mutex_unlock(&globalmutex);
+}
+
+/* Release in opposite order to kernel_fork_prepare(). */
+void
+kernel_fork_child()
+{
+	(void) pthread_mutex_unlock(
+	    &ses_delay_freed.ses_to_be_free_mutex);
+	(void) pthread_mutex_unlock(
+	    &obj_delay_freed.obj_to_be_free_mutex);
+	kernel_release_all_slots_mutexes();
+	(void) pthread_mutex_unlock(&globalmutex);
+}
+
+/*
  * PKCS#11 states that C_CancelFunction should always return
  * CKR_FUNCTION_NOT_PARALLEL
  */
diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
index 9f1520fcbb..d2164a3a0a 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -172,6 +172,9 @@ void kernel_delete_session(CK_SLOT_ID slotID, kernel_session_t *sp,
 
 void kernel_session_delay_free(kernel_session_t *sp);
 
+void kernel_acquire_all_slots_mutexes();
+void kernel_release_all_slots_mutexes();
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSessionUtil.c b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSessionUtil.c
index 81136e63af..aa0e6d6f88 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSessionUtil.c
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSessionUtil.c
@@ -253,9 +253,17 @@ kernel_delete_session(CK_SLOT_ID slotID, kernel_session_t *session_p,
 	 * count is not zero. This means that if the thread that is attempting
 	 * to close the session must wait until the prior operations on this
 	 * session are finished.
+	 *
+	 * Unless we are being forced to shut everything down, this only
+	 * happens if the library's _fini() is running not if someone
+	 * explicitly called C_Finalize().
 	 */
 	(void) pthread_mutex_lock(&session_p->ses_free_mutex);
 
+	if (wrapper_only) {
+		session_p->ses_refcnt = 0;
+	}
+
 	while (session_p->ses_refcnt != 0) {
 		/*
 		 * We set the SESSION_REFCNT_WAITING flag before we put
@@ -458,3 +466,58 @@ kernel_session_delay_free(kernel_session_t *sp)
 	}
 	(void) pthread_mutex_unlock(&ses_delay_freed.ses_to_be_free_mutex);
 }
+
+/*
+ * Acquire all slots' mutexes and all their sessions' mutexes.
+ * Order:
+ * 1. delete_sessions_mutex
+ * for each slot:
+ *  2. pslot->sl_mutex
+ *  for each session:
+ *   3. session_p->session_mutex
+ *   4. session_p->ses_free_mutex
+ */
+void
+kernel_acquire_all_slots_mutexes()
+{
+	int slotID;
+	kernel_slot_t *pslot;
+	kernel_session_t *session_p;
+
+	(void) pthread_mutex_lock(&delete_sessions_mutex);
+	for (slotID = 0; slotID < slot_count; slotID++) {
+		pslot = slot_table[slotID];
+		(void) pthread_mutex_lock(&pslot->sl_mutex);
+
+		/* Iterate through sessions acquiring all mutexes */
+		session_p = pslot->sl_sess_list;
+		while (session_p) {
+			(void) pthread_mutex_lock(&session_p->session_mutex);
+			session_p = session_p->next;
+		}
+	}
+}
+
+/* Release in opposite order to kernel_acquire_all_slots_mutexes(). */
+void
+kernel_release_all_slots_mutexes()
+{
+	int slotID;
+	kernel_slot_t *pslot;
+	kernel_session_t *session_p;
+
+	for (slotID = 0; slotID < slot_count; slotID++) {
+		pslot = slot_table[slotID];
+
+		/* Iterate through sessions releasing all mutexes */
+		session_p = pslot->sl_sess_list;
+		while (session_p) {
+			(void) pthread_mutex_unlock(&session_p->session_mutex);
+			session_p = session_p->next;
+		}
+
+		(void) pthread_mutex_unlock(&pslot->sl_mutex);
+	}
+
+	(void) pthread_mutex_unlock(&delete_sessions_mutex);
+}
diff --git a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softGeneral.c b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softGeneral.c
index a6ccf8e5d8..86a9576cc3 100644
--- a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softGeneral.c
+++ b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softGeneral.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -112,6 +112,7 @@ static struct CK_FUNCTION_LIST functionList = {
 };
 
 boolean_t softtoken_initialized = B_FALSE;
+static boolean_t softtoken_atfork_initialized = B_FALSE;
 
 static pid_t softtoken_pid = 0;
 
@@ -134,6 +135,9 @@ pthread_mutex_t soft_giant_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static CK_RV finalize_common(boolean_t force, CK_VOID_PTR pReserved);
 static void softtoken_fini();
+static void softtoken_fork_prepare();
+static void softtoken_fork_parent();
+static void softtoken_fork_child();
 
 CK_RV
 C_Initialize(CK_VOID_PTR pInitArgs)
@@ -244,6 +248,13 @@ C_Initialize(CK_VOID_PTR pInitArgs)
 		return (CKR_CANT_LOCK);
 	}
 
+	/* Children inherit parent's atfork handlers */
+	if (!softtoken_atfork_initialized) {
+		(void) pthread_atfork(softtoken_fork_prepare,
+		    softtoken_fork_parent, softtoken_fork_child);
+		softtoken_atfork_initialized = B_TRUE;
+	}
+
 	(void) pthread_mutex_unlock(&soft_giant_mutex);
 
 	/* Initialize the object_to_be_freed list */
@@ -480,6 +491,55 @@ looping_write(int fd, void *buf, int len)
 }
 
 /*
+ * Take out all mutexes before fork.
+ * Order:
+ * 1. soft_giant_mutex
+ * 2. soft_sessionlist_mutex
+ * 3. soft_slot.slot_mutex
+ * 4. soft_slot.keystore_mutex
+ * 5. all soft_session_list mutexs via soft_acquire_all_session_mutexes()
+ * 6. obj_delay_freed.obj_to_be_free_mutex;
+ * 7. ses_delay_freed.ses_to_be_free_mutex
+ */
+void
+softtoken_fork_prepare()
+{
+	(void) pthread_mutex_lock(&soft_giant_mutex);
+	(void) pthread_mutex_lock(&soft_sessionlist_mutex);
+	(void) pthread_mutex_lock(&soft_slot.slot_mutex);
+	(void) pthread_mutex_lock(&soft_slot.keystore_mutex);
+	soft_acquire_all_session_mutexes();
+	(void) pthread_mutex_lock(&obj_delay_freed.obj_to_be_free_mutex);
+	(void) pthread_mutex_lock(&ses_delay_freed.ses_to_be_free_mutex);
+}
+
+/* Release in opposite order to softtoken_fork_prepare(). */
+void
+softtoken_fork_parent()
+{
+	(void) pthread_mutex_unlock(&ses_delay_freed.ses_to_be_free_mutex);
+	(void) pthread_mutex_unlock(&obj_delay_freed.obj_to_be_free_mutex);
+	soft_release_all_session_mutexes();
+	(void) pthread_mutex_unlock(&soft_slot.keystore_mutex);
+	(void) pthread_mutex_unlock(&soft_slot.slot_mutex);
+	(void) pthread_mutex_unlock(&soft_sessionlist_mutex);
+	(void) pthread_mutex_unlock(&soft_giant_mutex);
+}
+
+/* Release in opposite order to softtoken_fork_prepare(). */
+void
+softtoken_fork_child()
+{
+	(void) pthread_mutex_unlock(&ses_delay_freed.ses_to_be_free_mutex);
+	(void) pthread_mutex_unlock(&obj_delay_freed.obj_to_be_free_mutex);
+	soft_release_all_session_mutexes();
+	(void) pthread_mutex_unlock(&soft_slot.keystore_mutex);
+	(void) pthread_mutex_unlock(&soft_slot.slot_mutex);
+	(void) pthread_mutex_unlock(&soft_sessionlist_mutex);
+	(void) pthread_mutex_unlock(&soft_giant_mutex);
+}
+
+/*
  * Perform a read that can handle EINTR.
  */
 int
diff --git a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSession.h b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSession.h
index b13181daf4..a95d8bc8c5 100644
--- a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSession.h
+++ b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSession.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -201,6 +201,9 @@ CK_RV soft_login(CK_UTF8CHAR_PTR pPin, CK_ULONG ulPinLen);
 
 void soft_logout(void);
 
+void soft_acquire_all_session_mutexes();
+void soft_release_all_session_mutexes();
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSessionUtil.c b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSessionUtil.c
index 4acff8d5ec..6c65da34e2 100644
--- a/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSessionUtil.c
+++ b/usr/src/lib/pkcs11/pkcs11_softtoken/common/softSessionUtil.c
@@ -711,3 +711,31 @@ soft_logout(void)
 	return;
 
 }
+
+void
+soft_acquire_all_session_mutexes()
+{
+	soft_session_t *session_p = soft_session_list;
+
+	/* Iterate through sessions acquiring all mutexes */
+	while (session_p) {
+		(void) pthread_mutex_lock(&session_p->session_mutex);
+		session_p = session_p->next;
+	}
+}
+
+void
+soft_release_all_session_mutexes()
+{
+	soft_session_t *session_p = soft_session_list;
+
+	/* Iterate through sessions releasing all mutexes */
+	while (session_p) {
+		/*
+		 * N.B. Ideally, should go in opposite order to guarantee
+		 * lock-order requirements but there is no tail pointer.
+		 */
+		(void) pthread_mutex_unlock(&session_p->session_mutex);
+		session_p = session_p->next;
+	}
+}