6579499 update operations should be able to use in-place transformations

5 files changed, 64 insertions, 21 deletions

diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c
index 826fb8364c..57b5cbae3a 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelDecrypt.c
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <pthread.h>
 #include <stdlib.h>
 #include <errno.h>
@@ -99,7 +97,13 @@ kernel_decrypt_init(kernel_session_t *session_p, kernel_object_t *key_p,
 
 	decrypt_init.di_session = session_p->k_session;
 	session_p->decrypt.mech = *pMechanism;
+
+	/* Cache this capability value for efficiency */
+	if (INPLACE_MECHANISM(session_p->decrypt.mech.mechanism)) {
+		session_p->decrypt.flags |= CRYPTO_OPERATION_INPLACE_OK;
+	}
 	(void) pthread_mutex_unlock(&session_p->session_mutex);
+
 	ses_lock_held = B_FALSE;
 	decrypt_init.di_mech.cm_type = k_mech_type;
 	decrypt_init.di_mech.cm_param = pMechanism->pParameter;
@@ -223,7 +227,8 @@ kernel_decrypt(kernel_session_t *session_p, CK_BYTE_PTR pEncryptedData,
 	 * some applications use a plaintext buffer that is larger
 	 * than it needs to be. We fix that here.
 	 */
-	inplace = INPLACE_MECHANISM(session_p->decrypt.mech.mechanism);
+	inplace = (session_p->decrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
+
 	if (ulEncryptedData < *pulDataLen && inplace) {
 		decrypt.cd_datalen = ulEncryptedData;
 	} else {
@@ -235,8 +240,9 @@ kernel_decrypt(kernel_session_t *session_p, CK_BYTE_PTR pEncryptedData,
 	decrypt.cd_databuf = (char *)pData;
 	decrypt.cd_encrlen = ulEncryptedData;
 	decrypt.cd_encrbuf = (char *)pEncryptedData;
-	decrypt.cd_flags = inplace && pData != NULL &&
-	    decrypt.cd_datalen == decrypt.cd_encrlen ?
+	decrypt.cd_flags = 
+	    ((inplace && (pData != NULL)) || (pData == pEncryptedData)) &&
+	    (decrypt.cd_datalen == decrypt.cd_encrlen) ?
 	    CRYPTO_INPLACE_OPERATION : 0;
 
 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT, &decrypt)) < 0) {
@@ -324,6 +330,7 @@ C_DecryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedPart,
 	CK_RV rv;
 	kernel_session_t *session_p;
 	boolean_t ses_lock_held = B_FALSE;
+	boolean_t inplace;
 	crypto_decrypt_update_t decrypt_update;
 	int r;
 
@@ -374,6 +381,12 @@ C_DecryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pEncryptedPart,
 	decrypt_update.du_encrlen = ulEncryptedPartLen;
 	decrypt_update.du_encrbuf = (char *)pEncryptedPart;
 
+	inplace = (session_p->decrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
+	decrypt_update.du_flags =
+	    ((inplace && (pPart != NULL)) || (pPart == pEncryptedPart)) &&
+	    (decrypt_update.du_datalen == decrypt_update.du_encrlen) ?
+	    CRYPTO_INPLACE_OPERATION : 0;
+
 	while ((r = ioctl(kernel_fd, CRYPTO_DECRYPT_UPDATE,
 	    &decrypt_update)) < 0) {
 		if (errno != EINTR)
diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c
index acd97161b6..d27945f877 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelEncrypt.c
@@ -19,12 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <pthread.h>
 #include <stdlib.h>
 #include <errno.h>
@@ -118,7 +116,13 @@ C_EncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMechanism,
 
 	encrypt_init.ei_session = session_p->k_session;
 	session_p->encrypt.mech = *pMechanism;
+
+	/* Cache this capability value for efficiency */
+	if (INPLACE_MECHANISM(session_p->encrypt.mech.mechanism)) {
+		session_p->encrypt.flags |= CRYPTO_OPERATION_INPLACE_OK;
+	}
 	(void) pthread_mutex_unlock(&session_p->session_mutex);
+
 	ses_lock_held = B_FALSE;
 	encrypt_init.ei_mech.cm_type = k_mech_type;
 	encrypt_init.ei_mech.cm_param = pMechanism->pParameter;
@@ -228,7 +232,7 @@ C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
 	 * some applications use a ciphertext buffer that is larger
 	 * than it needs to be. We fix that here.
 	 */
-	inplace = INPLACE_MECHANISM(session_p->encrypt.mech.mechanism);
+	inplace = (session_p->encrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
 	if (ulDataLen < *pulEncryptedDataLen && inplace) {
 		encrypt.ce_encrlen = ulDataLen;
 	} else {
@@ -240,8 +244,10 @@ C_Encrypt(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pData, CK_ULONG ulDataLen,
 	encrypt.ce_datalen = ulDataLen;
 	encrypt.ce_databuf = (char *)pData;
 	encrypt.ce_encrbuf = (char *)pEncryptedData;
-	encrypt.ce_flags = inplace && pEncryptedData != NULL &&
-	    encrypt.ce_encrlen == encrypt.ce_datalen ?
+	encrypt.ce_flags =
+	    ((inplace && (pEncryptedData != NULL)) ||
+	    (pData == pEncryptedData)) &&
+	    (encrypt.ce_encrlen == encrypt.ce_datalen) ?
 	    CRYPTO_INPLACE_OPERATION : 0;
 
 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT, &encrypt)) < 0) {
@@ -293,6 +299,7 @@ C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
 	CK_RV rv;
 	kernel_session_t *session_p;
 	boolean_t ses_lock_held = B_FALSE;
+	boolean_t inplace;
 	crypto_encrypt_update_t encrypt_update;
 	int r;
 
@@ -343,6 +350,13 @@ C_EncryptUpdate(CK_SESSION_HANDLE hSession, CK_BYTE_PTR pPart,
 	encrypt_update.eu_encrlen = *pulEncryptedPartLen;
 	encrypt_update.eu_encrbuf = (char *)pEncryptedPart;
 
+	inplace = (session_p->encrypt.flags & CRYPTO_OPERATION_INPLACE_OK) != 0;
+	encrypt_update.eu_flags =
+	    ((inplace && (pEncryptedPart != NULL)) ||
+	    (pPart == pEncryptedPart)) &&
+	    (encrypt_update.eu_encrlen == encrypt_update.eu_datalen) ?
+	    CRYPTO_INPLACE_OPERATION : 0;
+
 	while ((r = ioctl(kernel_fd, CRYPTO_ENCRYPT_UPDATE,
 	    &encrypt_update)) < 0) {
 		if (errno != EINTR)
diff --git a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
index d2164a3a0a..c95f5cf62b 100644
--- a/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
+++ b/usr/src/lib/pkcs11/pkcs11_kernel/common/kernelSession.h
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _KERNELSESSION_H
 #define	_KERNELSESSION_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -70,6 +68,7 @@ typedef struct digest_buf {
 #define	CRYPTO_EMULATE_USING_SW	0x00000008 /* ... use software */
 #define	CRYPTO_EMULATE_UPDATE_DONE 0x00000010 /* did at least one update */
 #define	CRYPTO_EMULATE_INIT_DONE 0x00000020 /* did init */
+#define	CRYPTO_OPERATION_INPLACE_OK 0x00000040 /* INPLACE_MECHANISM is true */
 
 typedef struct session {
 	CK_ULONG	magic_marker;	/* magic # be validated for integrity */
diff --git a/usr/src/uts/common/crypto/io/crypto.c b/usr/src/uts/common/crypto/io/crypto.c
index a9afc05149..ac9e37efd7 100644
--- a/usr/src/uts/common/crypto/io/crypto.c
+++ b/usr/src/uts/common/crypto/io/crypto.c
@@ -2806,11 +2806,12 @@ cipher(dev_t dev, caddr_t arg, int mode,
 	ctxpp = (single == crypto_encrypt_single) ?
 	    &sp->sd_encr_ctx : &sp->sd_decr_ctx;
 
-	/* in-place is specified by setting output NULL */
 	if (do_inplace)
+		/* specify in-place buffers with output = NULL */
 		rv = (single)(*ctxpp, &encr, NULL, NULL);
 	else
 		rv = (single)(*ctxpp, &data, &encr, NULL);
+
 	if (KCF_CONTEXT_DONE(rv))
 		*ctxpp = NULL;
 
@@ -2886,6 +2887,7 @@ cipher_update(dev_t dev, caddr_t arg, int mode,
 	crypto_ctx_t **ctxpp;
 	crypto_data_t data, encr;
 	size_t datalen, encrlen, need = 0;
+	boolean_t do_inplace;
 	char *encrbuf;
 	int error = 0;
 	int rv;
@@ -2933,7 +2935,9 @@ cipher_update(dev_t dev, caddr_t arg, int mode,
 		goto out;
 	}
 
-	need = datalen + encrlen;
+	do_inplace = (STRUCT_FGET(encrypt_update, eu_flags) &
+	    CRYPTO_INPLACE_OPERATION) != 0;
+	need = do_inplace ? datalen : datalen + encrlen;
 
 	if ((rv = CRYPTO_BUFFER_CHECK(sp, need, rctl_chk)) !=
 	    CRYPTO_SUCCESS) {
@@ -2950,12 +2954,21 @@ cipher_update(dev_t dev, caddr_t arg, int mode,
 		goto out;
 	}
 
-	INIT_RAW_CRYPTO_DATA(encr, encrlen);
+	if (do_inplace) {
+		/* specify in-place buffers with output = input */
+		encr = data;
+	} else {
+		INIT_RAW_CRYPTO_DATA(encr, encrlen);
+	}
 
 	ctxpp = (update == crypto_encrypt_update) ?
 	    &sp->sd_encr_ctx : &sp->sd_decr_ctx;
 
-	rv = (update)(*ctxpp, &data, &encr, NULL);
+	if (do_inplace)
+		/* specify in-place buffers with output = NULL */
+		rv = (update)(*ctxpp, &encr, NULL, NULL);
+	else
+		rv = (update)(*ctxpp, &data, &encr, NULL);
 
 	if (rv == CRYPTO_SUCCESS || rv == CRYPTO_BUFFER_TOO_SMALL) {
 		if (rv == CRYPTO_SUCCESS) {
@@ -2987,7 +3000,7 @@ out:
 	if (data.cd_raw.iov_base != NULL)
 		kmem_free(data.cd_raw.iov_base, datalen);
 
-	if (encr.cd_raw.iov_base != NULL)
+	if (!do_inplace && (encr.cd_raw.iov_base != NULL))
 		kmem_free(encr.cd_raw.iov_base, encrlen);
 
 	if (error != 0)
diff --git a/usr/src/uts/common/sys/crypto/ioctl.h b/usr/src/uts/common/sys/crypto/ioctl.h
index ca5d748a99..74fab17ca1 100644
--- a/usr/src/uts/common/sys/crypto/ioctl.h
+++ b/usr/src/uts/common/sys/crypto/ioctl.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -271,6 +271,7 @@ typedef struct crypto_encrypt_update {
 	caddr_t			eu_databuf;
 	size_t			eu_encrlen;
 	caddr_t			eu_encrbuf;
+	uint_t			eu_flags;
 } crypto_encrypt_update_t;
 
 typedef struct crypto_encrypt_final {
@@ -304,6 +305,7 @@ typedef struct crypto_decrypt_update {
 	caddr_t			du_encrbuf;
 	size_t			du_datalen;
 	caddr_t			du_databuf;
+	uint_t			du_flags;
 } crypto_decrypt_update_t;
 
 typedef struct crypto_decrypt_final {
@@ -541,6 +543,7 @@ typedef struct crypto_encrypt_update32 {
 	caddr32_t		eu_databuf;
 	size32_t		eu_encrlen;
 	caddr32_t		eu_encrbuf;
+	uint_t			eu_flags;
 } crypto_encrypt_update32_t;
 
 typedef struct crypto_encrypt_final32 {
@@ -582,6 +585,7 @@ typedef struct crypto_decrypt_update32 {
 	caddr32_t		du_encrbuf;
 	size32_t		du_datalen;
 	caddr32_t		du_databuf;
+	uint_t			du_flags;
 } crypto_decrypt_update32_t;
 
 typedef struct crypto_decrypt_final32 {