6746735 PKCS#11 engine should use extended FILE space API

6731380 return codes of several functions are not checked in the PKCS#11 engine code

2 files changed, 40 insertions, 11 deletions

diff --git a/usr/src/common/openssl/crypto/engine/hw_pk11.c b/usr/src/common/openssl/crypto/engine/hw_pk11.c
index fd6ca79dfd..5030d5c477 100644
--- a/usr/src/common/openssl/crypto/engine/hw_pk11.c
+++ b/usr/src/common/openssl/crypto/engine/hw_pk11.c
@@ -890,7 +890,8 @@ static int bind_pk11(ENGINE *e)
 	RSA_METHOD *pk11_rsa = PK11_RSA();
 #endif	/* OPENSSL_NO_RSA */
 	if (!pk11_library_initialized)
-		(void) pk11_library_init(e);
+		if (!pk11_library_init(e))
+			return (0);
 
 	if (!ENGINE_set_id(e, engine_pk11_id) ||
 	    !ENGINE_set_name(e, engine_pk11_name) ||
@@ -1804,7 +1805,8 @@ static int pk11_setup_session(PK11_SESSION *sp, PK11_OPTYPE optype)
 		 * reinitialize of the session
 		 */
 		pk11_library_initialized = FALSE;
-		(void) pk11_library_init(NULL);
+		if (!pk11_library_init(NULL))
+			return (0);
 		rv = pFuncList->C_OpenSession(myslot, CKF_SERIAL_SESSION,
 			NULL_PTR, NULL_PTR, &sp->session);
 		}
@@ -2842,11 +2844,11 @@ pk11_digest_cleanup(EVP_MD_CTX *ctx)
 		 * If state->sp is not NULL then pk11_digest_final() has not
 		 * been called yet. We must call it now to free any memory
 		 * that might have been allocated in the token when
-		 * pk11_digest_init() was called.
+		 * pk11_digest_init() was called. pk11_digest_final()
+		 * will return the session to the cache.
 		 */
-		(void) pk11_digest_final(ctx, buf);
-		pk11_return_session(state->sp, OP_DIGEST);
-		state->sp = NULL;
+		if (!pk11_digest_final(ctx, buf))
+			return (0);
 		}
 
 	return (1);
diff --git a/usr/src/common/openssl/crypto/engine/hw_pk11_pub.c b/usr/src/common/openssl/crypto/engine/hw_pk11_pub.c
index afe7bc8771..2487fef1a2 100644
--- a/usr/src/common/openssl/crypto/engine/hw_pk11_pub.c
+++ b/usr/src/common/openssl/crypto/engine/hw_pk11_pub.c
@@ -176,6 +176,13 @@ static int check_new_dh_key(PK11_SESSION *sp, DH *dh);
 static int init_template_value(BIGNUM *bn, CK_VOID_PTR *pValue,
 	CK_ULONG *ulValueLen);
 
+/* Read mode string to be used for fopen() */
+#if SOLARIS_OPENSSL
+static char *read_mode_flags = "rF";
+#else
+static char *read_mode_flags = "r";
+#endif
+
 /*
  * increment/create reference for an asymmetric key handle via active list
  * manipulation. If active list operation fails, unlock (if locked), set error
@@ -1025,7 +1032,7 @@ static int pk11_RSA_sign(int type, const unsigned char *m, unsigned int m_len,
 		}
 
 	j = RSA_size(rsa);
-	if ((i-RSA_PKCS1_PADDING) > j)
+	if ((i - RSA_PKCS1_PADDING) > j)
 		{
 		PK11err(PK11_F_RSA_SIGN, PK11_R_DIGEST_TOO_BIG);
 		goto err;
@@ -1145,7 +1152,7 @@ static int pk11_RSA_verify(int type, const unsigned char *m,
 		}
 
 	j = RSA_size(rsa);
-	if ((i-RSA_PKCS1_PADDING) > j)
+	if ((i - RSA_PKCS1_PADDING) > j)
 		{
 		PK11err(PK11_F_RSA_VERIFY, PK11_R_DIGEST_TOO_BIG);
 		goto err;
@@ -1160,7 +1167,7 @@ static int pk11_RSA_verify(int type, const unsigned char *m,
 			goto err;
 			}
 		p = s;
-		i2d_X509_SIG(&sig, &p);
+		(void) i2d_X509_SIG(&sig, &p);
 		}
 
 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
@@ -1222,7 +1229,7 @@ EVP_PKEY *pk11_load_privkey(ENGINE* e, const char *privkey_file,
 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
 		return (NULL);
 
-	if ((pubkey = fopen(privkey_file, "r")) != NULL)
+	if ((pubkey = fopen(privkey_file, read_mode_flags)) != NULL)
 		{
 		pkey = PEM_read_PrivateKey(pubkey, NULL, NULL, NULL);
 		(void) fclose(pubkey);
@@ -1269,7 +1276,7 @@ EVP_PKEY *pk11_load_pubkey(ENGINE* e, const char *pubkey_file,
 	if ((sp = pk11_get_session(OP_RSA)) == NULL)
 		return (NULL);
 
-	if ((pubkey = fopen(pubkey_file, "r")) != NULL)
+	if ((pubkey = fopen(pubkey_file, read_mode_flags)) != NULL)
 		{
 		pkey = PEM_read_PUBKEY(pubkey, NULL, NULL, NULL);
 		(void) fclose(pubkey);
@@ -1423,6 +1430,10 @@ static CK_OBJECT_HANDLE pk11_get_public_rsa_key(RSA* rsa,
 err:
 	if (rollback)
 		{
+		/*
+		 * We do not care about the return value from C_DestroyObject()
+		 * since we are doing rollback.
+		 */
 		if (found == 0)
 			(void) pFuncList->C_DestroyObject(session, h_key);
 		h_key = CK_INVALID_HANDLE;
@@ -1561,6 +1572,10 @@ static CK_OBJECT_HANDLE pk11_get_private_rsa_key(RSA* rsa,
 err:
 	if (rollback)
 		{
+		/*
+		 * We do not care about the return value from C_DestroyObject()
+		 * since we are doing rollback.
+		 */
 		if (found == 0)
 			(void) pFuncList->C_DestroyObject(session, h_key);
 		h_key = CK_INVALID_HANDLE;
@@ -1962,6 +1977,10 @@ static CK_OBJECT_HANDLE pk11_get_public_dsa_key(DSA* dsa,
 err:
 	if (rollback)
 		{
+		/*
+		 * We do not care about the return value from C_DestroyObject()
+		 * since we are doing rollback.
+		 */
 		if (found == 0)
 			(void) pFuncList->C_DestroyObject(session, h_key);
 		h_key = CK_INVALID_HANDLE;
@@ -2087,6 +2106,10 @@ static CK_OBJECT_HANDLE pk11_get_private_dsa_key(DSA* dsa,
 err:
 	if (rollback)
 		{
+		/*
+		 * We do not care about the return value from C_DestroyObject()
+		 * since we are doing rollback.
+		 */
 		if (found == 0)
 			(void) pFuncList->C_DestroyObject(session, h_key);
 		h_key = CK_INVALID_HANDLE;
@@ -2702,6 +2725,10 @@ static CK_OBJECT_HANDLE pk11_get_dh_key(DH* dh,
 err:
 	if (rollback)
 		{
+		/*
+		 * We do not care about the return value from C_DestroyObject()
+		 * since we are doing rollback.
+		 */
 		if (found == 0)
 			(void) pFuncList->C_DestroyObject(session, h_key);
 		h_key = CK_INVALID_HANDLE;