2426 wanboot openssl needs to not call cast function pointers

Reviewed by: Garrett D'Amore <garrett@damore.org>
Reviewed by: Gordon Ross <gwr@nexenta.com>
Reviewed by: Joshua M. Clulow <josh@sysmgr.org>
Approved by: Gordon Ross <gwr@nexenta.com>

3 files changed, 66 insertions, 25 deletions

diff --git a/usr/src/common/openssl/crypto/asn1/asn1.h b/usr/src/common/openssl/crypto/asn1/asn1.h
index 8781bef3bb..bc77e0544d 100644
--- a/usr/src/common/openssl/crypto/asn1/asn1.h
+++ b/usr/src/common/openssl/crypto/asn1/asn1.h
@@ -321,6 +321,17 @@ typedef struct ASN1_VALUE_st ASN1_VALUE;
 #define I2D_OF(type) int (*)(type *,unsigned char **)
 #define I2D_OF_const(type) int (*)(const type *,unsigned char **)
 
+#define CHECKED_D2I_OF(type, d2i) \
+    ((d2i_of_void*) (1 ? d2i : ((D2I_OF(type))0)))
+#define CHECKED_I2D_OF(type, i2d) \
+    ((i2d_of_void*) (1 ? i2d : ((I2D_OF(type))0)))
+#define CHECKED_NEW_OF(type, xnew) \
+    ((void *(*)(void)) (1 ? xnew : ((type *(*)(void))0)))
+#define CHECKED_PTR_OF(type, p) \
+    ((void*) (1 ? p : (type*)0))
+#define CHECKED_PPTR_OF(type, p) \
+    ((void**) (1 ? p : (type**)0))
+
 #define TYPEDEF_D2I_OF(type) typedef type *d2i_of_##type(type **,const unsigned char **,long)
 #define TYPEDEF_I2D_OF(type) typedef int i2d_of_##type(type *,unsigned char **)
 #define TYPEDEF_D2I2D_OF(type) TYPEDEF_D2I_OF(type); TYPEDEF_I2D_OF(type)
@@ -901,22 +912,38 @@ int ASN1_object_size(int constructed, int length, int tag);
 /* Used to implement other functions */
 void *ASN1_dup(i2d_of_void *i2d, d2i_of_void *d2i, char *x);
 #define ASN1_dup_of(type,i2d,d2i,x) \
-	((type *(*)(I2D_OF(type),D2I_OF(type),type *))openssl_fcast(ASN1_dup))(i2d,d2i,x)
+     ((type*)ASN1_dup(CHECKED_I2D_OF(type, i2d), \
+                     CHECKED_D2I_OF(type, d2i), \
+                     CHECKED_PTR_OF(type, x)))
+ 
 #define ASN1_dup_of_const(type,i2d,d2i,x) \
-	((type *(*)(I2D_OF_const(type),D2I_OF(type),type *))openssl_fcast(ASN1_dup))(i2d,d2i,x)
+     ((type*)ASN1_dup(CHECKED_I2D_OF(const type, i2d), \
+                     CHECKED_D2I_OF(type, d2i), \
+                     CHECKED_PTR_OF(const type, x)))
 
 void *ASN1_item_dup(const ASN1_ITEM *it, void *x);
 
 #ifndef OPENSSL_NO_FP_API
 void *ASN1_d2i_fp(void *(*xnew)(void), d2i_of_void *d2i, FILE *in, void **x);
-#define ASN1_d2i_fp_of(type,xnew,d2i,in,x) \
-	((type *(*)(type *(*)(void),D2I_OF(type),FILE *,type **))openssl_fcast(ASN1_d2i_fp))(xnew,d2i,in,x)
+
+#define ASN1_d2i_fp_of(type,xnew,d2i,in,x)                              \
+     ((type*)ASN1_d2i_fp(CHECKED_NEW_OF(type, xnew), \
+                        CHECKED_D2I_OF(type, d2i), \
+                        in, \
+                        CHECKED_PPTR_OF(type, x)))
+
 void *ASN1_item_d2i_fp(const ASN1_ITEM *it, FILE *in, void *x);
 int ASN1_i2d_fp(i2d_of_void *i2d,FILE *out,void *x);
 #define ASN1_i2d_fp_of(type,i2d,out,x) \
-	((int (*)(I2D_OF(type),FILE *,type *))openssl_fcast(ASN1_i2d_fp))(i2d,out,x)
-#define ASN1_i2d_fp_of_const(type,i2d,out,x) \
-	((int (*)(I2D_OF_const(type),FILE *,type *))openssl_fcast(ASN1_i2d_fp))(i2d,out,x)
+     (ASN1_i2d_fp(CHECKED_I2D_OF(type, i2d), \
+                 out, \
+                 CHECKED_PTR_OF(type, x)))
+
+#define ASN1_i2d_fp_of_const(type,i2d,out,x)                            \
+     (ASN1_i2d_fp(CHECKED_I2D_OF(const type, i2d), \
+                 out, \
+                 CHECKED_PTR_OF(const type, x)))
+
 int ASN1_item_i2d_fp(const ASN1_ITEM *it, FILE *out, void *x);
 int ASN1_STRING_print_ex_fp(FILE *fp, ASN1_STRING *str, unsigned long flags);
 #endif
@@ -926,13 +953,24 @@ int ASN1_STRING_to_UTF8(unsigned char **out, ASN1_STRING *in);
 #ifndef OPENSSL_NO_BIO
 void *ASN1_d2i_bio(void *(*xnew)(void), d2i_of_void *d2i, BIO *in, void **x);
 #define ASN1_d2i_bio_of(type,xnew,d2i,in,x) \
-	((type *(*)(type *(*)(void),D2I_OF(type),BIO *,type **))openssl_fcast(ASN1_d2i_bio))(xnew,d2i,in,x)
+     ((type*)ASN1_d2i_bio( CHECKED_NEW_OF(type, xnew), \
+                          CHECKED_D2I_OF(type, d2i), \
+                          in, \
+                          CHECKED_PPTR_OF(type, x)))
+
 void *ASN1_item_d2i_bio(const ASN1_ITEM *it, BIO *in, void *x);
 int ASN1_i2d_bio(i2d_of_void *i2d,BIO *out, unsigned char *x);
 #define ASN1_i2d_bio_of(type,i2d,out,x) \
-	((int (*)(I2D_OF(type),BIO *,type *))openssl_fcast(ASN1_i2d_bio))(i2d,out,x)
+     (ASN1_i2d_bio(CHECKED_I2D_OF(type, i2d), \
+                  out, \
+                  CHECKED_PTR_OF(type, x)))
+
 #define ASN1_i2d_bio_of_const(type,i2d,out,x) \
-	((int (*)(I2D_OF_const(type),BIO *,const type *))openssl_fcast(ASN1_i2d_bio))(i2d,out,x)
+     (ASN1_i2d_bio(CHECKED_I2D_OF(const type, i2d), \
+                  out, \
+                  CHECKED_PTR_OF(const type, x)))
+ 
+
 int ASN1_item_i2d_bio(const ASN1_ITEM *it, BIO *out, void *x);
 int ASN1_UTCTIME_print(BIO *fp,ASN1_UTCTIME *a);
 int ASN1_GENERALIZEDTIME_print(BIO *fp,ASN1_GENERALIZEDTIME *a);
@@ -976,7 +1014,10 @@ void *ASN1_item_unpack(ASN1_STRING *oct, const ASN1_ITEM *it);
 ASN1_STRING *ASN1_pack_string(void *obj, i2d_of_void *i2d,
 			      ASN1_OCTET_STRING **oct);
 #define ASN1_pack_string_of(type,obj,i2d,oct) \
-	((ASN1_STRING *(*)(type *,I2D_OF(type),ASN1_OCTET_STRING **))openssl_fcast(ASN1_pack_string))(obj,i2d,oct)
+     (ASN1_pack_string(CHECKED_PTR_OF(type, obj), \
+                      CHECKED_I2D_OF(type, i2d), \
+                      oct))
+ 
 ASN1_STRING *ASN1_item_pack(void *obj, const ASN1_ITEM *it, ASN1_OCTET_STRING **oct);
 
 void ASN1_STRING_set_default_mask(unsigned long mask);
diff --git a/usr/src/common/openssl/crypto/pem/pem.h b/usr/src/common/openssl/crypto/pem/pem.h
index 7db6b423d0..9a50472ef8 100644
--- a/usr/src/common/openssl/crypto/pem/pem.h
+++ b/usr/src/common/openssl/crypto/pem/pem.h
@@ -220,19 +220,19 @@ typedef struct pem_ctx_st
 #define IMPLEMENT_PEM_read_fp(name, type, str, asn1) \
 type *PEM_read_##name(FILE *fp, type **x, pem_password_cb *cb, void *u)\
 { \
-return(((type *(*)(D2I_OF(type),char *,FILE *,type **,pem_password_cb *,void *))openssl_fcast(PEM_ASN1_read))(d2i_##asn1, str,fp,x,cb,u)); \
+return PEM_ASN1_read((d2i_of_void *)d2i_##asn1, str,fp,(void **)x,cb,u); \
 } \
 
 #define IMPLEMENT_PEM_write_fp(name, type, str, asn1) \
 int PEM_write_##name(FILE *fp, type *x) \
 { \
-return(((int (*)(I2D_OF(type),const char *,FILE *,type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write))(i2d_##asn1,str,fp,x,NULL,NULL,0,NULL,NULL)); \
+return PEM_ASN1_write((i2d_of_void *)i2d_##asn1,str,fp,x,NULL,NULL,0,NULL,NULL); \
 }
 
 #define IMPLEMENT_PEM_write_fp_const(name, type, str, asn1) \
 int PEM_write_##name(FILE *fp, const type *x) \
 { \
-return(((int (*)(I2D_OF_const(type),const char *,FILE *, const type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write))(i2d_##asn1,str,fp,x,NULL,NULL,0,NULL,NULL)); \
+return PEM_ASN1_write((i2d_of_void *)i2d_##asn1,str,fp,(void *)x,NULL,NULL,0,NULL,NULL); \
 }
 
 #define IMPLEMENT_PEM_write_cb_fp(name, type, str, asn1) \
@@ -240,7 +240,7 @@ int PEM_write_##name(FILE *fp, type *x, const EVP_CIPHER *enc, \
 	     unsigned char *kstr, int klen, pem_password_cb *cb, \
 		  void *u) \
 	{ \
-	return(((int (*)(I2D_OF(type),const char *,FILE *,type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write))(i2d_##asn1,str,fp,x,enc,kstr,klen,cb,u)); \
+	return PEM_ASN1_write((i2d_of_void *)i2d_##asn1,str,fp,x,enc,kstr,klen,cb,u); \
 	}
 
 #define IMPLEMENT_PEM_write_cb_fp_const(name, type, str, asn1) \
@@ -248,7 +248,7 @@ int PEM_write_##name(FILE *fp, type *x, const EVP_CIPHER *enc, \
 	     unsigned char *kstr, int klen, pem_password_cb *cb, \
 		  void *u) \
 	{ \
-	return(((int (*)(I2D_OF_const(type),const char *,FILE *,type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write))(i2d_##asn1,str,fp,x,enc,kstr,klen,cb,u)); \
+	return PEM_ASN1_write((i2d_of_void *)i2d_##asn1,str,fp,x,enc,kstr,klen,cb,u); \
 	}
 
 #endif
@@ -256,33 +256,33 @@ int PEM_write_##name(FILE *fp, type *x, const EVP_CIPHER *enc, \
 #define IMPLEMENT_PEM_read_bio(name, type, str, asn1) \
 type *PEM_read_bio_##name(BIO *bp, type **x, pem_password_cb *cb, void *u)\
 { \
-return(((type *(*)(D2I_OF(type),const char *,BIO *,type **,pem_password_cb *,void *))openssl_fcast(PEM_ASN1_read_bio))(d2i_##asn1, str,bp,x,cb,u)); \
+return PEM_ASN1_read_bio((d2i_of_void *)d2i_##asn1, str,bp,(void **)x,cb,u); \
 }
 
 #define IMPLEMENT_PEM_write_bio(name, type, str, asn1) \
 int PEM_write_bio_##name(BIO *bp, type *x) \
 { \
-return(((int (*)(I2D_OF(type),const char *,BIO *,type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write_bio))(i2d_##asn1,str,bp,x,NULL,NULL,0,NULL,NULL)); \
+return PEM_ASN1_write_bio((i2d_of_void *)i2d_##asn1,str,bp,x,NULL,NULL,0,NULL,NULL); \
 }
 
 #define IMPLEMENT_PEM_write_bio_const(name, type, str, asn1) \
 int PEM_write_bio_##name(BIO *bp, const type *x) \
 { \
-return(((int (*)(I2D_OF_const(type),const char *,BIO *,const type *, const EVP_CIPHER *,unsigned char *,int, pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write_bio))(i2d_##asn1,str,bp,x,NULL,NULL,0,NULL,NULL)); \
+return PEM_ASN1_write_bio((i2d_of_void *)i2d_##asn1,str,bp,(void *)x,NULL,NULL,0,NULL,NULL); \
 }
 
 #define IMPLEMENT_PEM_write_cb_bio(name, type, str, asn1) \
 int PEM_write_bio_##name(BIO *bp, type *x, const EVP_CIPHER *enc, \
 	     unsigned char *kstr, int klen, pem_password_cb *cb, void *u) \
 	{ \
-	return(((int (*)(I2D_OF(type),const char *,BIO *,type *,const EVP_CIPHER *,unsigned char *,int,pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write_bio))(i2d_##asn1,str,bp,x,enc,kstr,klen,cb,u)); \
+	return PEM_ASN1_write_bio((i2d_of_void *)i2d_##asn1,str,bp,x,enc,kstr,klen,cb,u); \
 	}
 
 #define IMPLEMENT_PEM_write_cb_bio_const(name, type, str, asn1) \
 int PEM_write_bio_##name(BIO *bp, type *x, const EVP_CIPHER *enc, \
 	     unsigned char *kstr, int klen, pem_password_cb *cb, void *u) \
 	{ \
-	return(((int (*)(I2D_OF_const(type),const char *,BIO *,type *,const EVP_CIPHER *,unsigned char *,int,pem_password_cb *,void *))openssl_fcast(PEM_ASN1_write_bio))(i2d_##asn1,str,bp,x,enc,kstr,klen,cb,u)); \
+	return PEM_ASN1_write_bio((i2d_of_void *)i2d_##asn1,str,bp,(void *)x,enc,kstr,klen,cb,u); \
 	}
 
 #define IMPLEMENT_PEM_write(name, type, str, asn1) \
@@ -547,7 +547,7 @@ void *	PEM_ASN1_read_bio(d2i_of_void *d2i, const char *name, BIO *bp,
 			  void **x, pem_password_cb *cb, void *u);
 #define PEM_ASN1_read_bio_of(type,d2i,name,bp,x,cb,u) \
 ((type *(*)(D2I_OF(type),const char *,BIO *,type **,pem_password_cb *,void *))openssl_fcast(PEM_ASN1_read_bio))(d2i,name,bp,x,cb,u)
-int	PEM_ASN1_write_bio(i2d_of_void *i2d,const char *name,BIO *bp,char *x,
+int	PEM_ASN1_write_bio(i2d_of_void *i2d,const char *name,BIO *bp,void *x,
 			   const EVP_CIPHER *enc,unsigned char *kstr,int klen,
 			   pem_password_cb *cb, void *u);
 #define PEM_ASN1_write_bio_of(type,i2d,name,bp,x,enc,kstr,klen,cb,u) \
@@ -565,7 +565,7 @@ int	PEM_write(FILE *fp,char *name,char *hdr,unsigned char *data,long len);
 void *  PEM_ASN1_read(d2i_of_void *d2i, const char *name, FILE *fp, void **x,
 		      pem_password_cb *cb, void *u);
 int	PEM_ASN1_write(i2d_of_void *i2d,const char *name,FILE *fp,
-		       char *x,const EVP_CIPHER *enc,unsigned char *kstr,
+		       void *x,const EVP_CIPHER *enc,unsigned char *kstr,
 		       int klen,pem_password_cb *callback, void *u);
 STACK_OF(X509_INFO) *	PEM_X509_INFO_read(FILE *fp, STACK_OF(X509_INFO) *sk,
 	pem_password_cb *cb, void *u);
diff --git a/usr/src/common/openssl/crypto/pem/pem_lib.c b/usr/src/common/openssl/crypto/pem/pem_lib.c
index 7cfc2f3e0a..47f6d4e379 100644
--- a/usr/src/common/openssl/crypto/pem/pem_lib.c
+++ b/usr/src/common/openssl/crypto/pem/pem_lib.c
@@ -261,7 +261,7 @@ err:
 
 #ifndef OPENSSL_NO_FP_API
 int PEM_ASN1_write(i2d_of_void *i2d, const char *name, FILE *fp,
-		   char *x, const EVP_CIPHER *enc, unsigned char *kstr,
+		   void *x, const EVP_CIPHER *enc, unsigned char *kstr,
 		   int klen, pem_password_cb *callback, void *u)
         {
         BIO *b;
@@ -280,7 +280,7 @@ int PEM_ASN1_write(i2d_of_void *i2d, const char *name, FILE *fp,
 #endif
 
 int PEM_ASN1_write_bio(i2d_of_void *i2d, const char *name, BIO *bp,
-		       char *x, const EVP_CIPHER *enc, unsigned char *kstr,
+		       void *x, const EVP_CIPHER *enc, unsigned char *kstr,
 		       int klen, pem_password_cb *callback, void *u)
 	{
 	EVP_CIPHER_CTX ctx;