13985 Support building with OpenSSL 3.0

Reviewed by: Andrew Stormont <andyjstormont@gmail.com>
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Reviewed by: Jason King <jason.brian.king@gmail.com>
Approved by: Dan McDonald <danmcd@joyent.com>

4 files changed, 23 insertions, 8 deletions

diff --git a/usr/src/lib/krb5/plugins/preauth/pkinit/pkinit_crypto_openssl.c b/usr/src/lib/krb5/plugins/preauth/pkinit/pkinit_crypto_openssl.c
index 1dc79baa0a..8a6ad8b387 100644
--- a/usr/src/lib/krb5/plugins/preauth/pkinit/pkinit_crypto_openssl.c
+++ b/usr/src/lib/krb5/plugins/preauth/pkinit/pkinit_crypto_openssl.c
@@ -4542,7 +4542,11 @@ pkinit_find_private_key(pkinit_identity_crypto_context id_cryptoctx,
     CK_ATTRIBUTE attrs[4];
     CK_ULONG count;
     CK_KEY_TYPE keytype;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    const RSA *rsa;
+#else
     RSA *rsa;
+#endif
     unsigned int nattrs = 0;
     int r;
 #ifdef PKINIT_USE_KEY_USAGE
diff --git a/usr/src/lib/libkmf/plugins/kmf_openssl/common/openssl_spi.c b/usr/src/lib/libkmf/plugins/kmf_openssl/common/openssl_spi.c
index 142fa13f4a..1ef0000651 100644
--- a/usr/src/lib/libkmf/plugins/kmf_openssl/common/openssl_spi.c
+++ b/usr/src/lib/libkmf/plugins/kmf_openssl/common/openssl_spi.c
@@ -140,7 +140,9 @@ static uchar_t G[] = { 0x00, 0x62, 0x6d, 0x02, 0x78, 0x39, 0xea, 0x0a,
  * Declare some new macros for managing stacks of EVP_PKEYS.
  */
 #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+/* BEGIN CSTYLED */
 DECLARE_STACK_OF(EVP_PKEY)
+/* END CSTYLED */
 
 #define	sk_EVP_PKEY_new_null() SKM_sk_new_null(EVP_PKEY)
 #define	sk_EVP_PKEY_free(st) SKM_sk_free(EVP_PKEY, (st))
@@ -151,7 +153,9 @@ DECLARE_STACK_OF(EVP_PKEY)
 	(free_func))
 
 #else
+/* BEGIN CSTYLED */
 DEFINE_STACK_OF(EVP_PKEY)
+/* END CSTYLED */
 #endif
 
 mutex_t init_lock = DEFAULTMUTEX;
@@ -1388,8 +1392,13 @@ ssl_write_key(KMF_HANDLE *kmfh, KMF_ENCODE_FORMAT format, BIO *out,
     KMF_CREDENTIAL *cred, EVP_PKEY *pkey, boolean_t private)
 {
 	int rv = 0;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	const RSA *rsa;
+	const DSA *dsa;
+#else
 	RSA *rsa;
 	DSA *dsa;
+#endif
 
 	if (pkey == NULL || out == NULL)
 		return (KMF_ERR_BAD_PARAMETER);
@@ -2066,8 +2075,13 @@ OpenSSL_CertGetPrintable(KMF_HANDLE_T handle, const KMF_DATA *pcert,
 
 	case KMF_CERT_PUBKEY_DATA:
 		{
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+			const RSA *rsa;
+			const DSA *dsa;
+#else
 			RSA *rsa;
 			DSA *dsa;
+#endif
 
 			EVP_PKEY *pkey = X509_get_pubkey(xcert);
 			if (pkey == NULL) {
diff --git a/usr/src/tools/smatch/Makefile b/usr/src/tools/smatch/Makefile
index ef67695404..2d0365c458 100644
--- a/usr/src/tools/smatch/Makefile
+++ b/usr/src/tools/smatch/Makefile
@@ -20,7 +20,7 @@
 #
 
 PROG = smatch
-SPARSE_VERSION = 0.6.1-rc1-il-5
+SPARSE_VERSION = 0.6.1-rc1-il-7
 
 include ../Makefile.tools
 
@@ -46,11 +46,11 @@ CFLAGS += -DGCC_BASE='"/no/such/dir"'
 CFLAGS += -DMULTIARCH_TRIPLET=NULL
 
 LDCHECKS = $(ZASSERTDEFLIB) $(ZGUIDANCE) $(ZFATALWARNINGS)
-LDLIBS += $(ZLAZYLOAD) $(ZDIRECT) -lsqlite3 -lcrypto -lgcc -lm -lc
+LDLIBS += $(ZLAZYLOAD) $(ZDIRECT) -lsqlite3 -lmd -lgcc -lm -lc
 LDFLAGS = $(MAPFILE.NES:%=-Wl,-M%)
 LDFLAGS += -L$(NATIVE_ADJUNCT)/lib -R$(NATIVE_ADJUNCT)/lib
 
-NATIVE_LIBS += libsqlite3.so libcrypto.so libm.so libc.so
+NATIVE_LIBS += libsqlite3.so libmd.so libm.so libc.so
 LDFLAGS += $(NATIVE_LIBS:%=$(ZASSERTDEFLIB)=%)
 
 CPPFLAGS += -nostdinc
diff --git a/usr/src/tools/smatch/src/smatch_mtag.c b/usr/src/tools/smatch/src/smatch_mtag.c
index fc20107140..a8ca03b4b2 100644
--- a/usr/src/tools/smatch/src/smatch_mtag.c
+++ b/usr/src/tools/smatch/src/smatch_mtag.c
@@ -46,7 +46,7 @@
 #include "smatch_slist.h"
 #include "smatch_extra.h"
 
-#include <openssl/md5.h>
+#include <md5.h>
 
 static int my_id;
 
@@ -54,13 +54,10 @@ mtag_t str_to_mtag(const char *str)
 {
 	unsigned char c[MD5_DIGEST_LENGTH];
 	unsigned long long *tag = (unsigned long long *)&c;
-	MD5_CTX mdContext;
 	int len;
 
 	len = strlen(str);
-	MD5_Init(&mdContext);
-	MD5_Update(&mdContext, str, len);
-	MD5_Final(c, &mdContext);
+	md5_calc(c, str, len);
 
 	*tag &= ~MTAG_ALIAS_BIT;
 	*tag &= ~MTAG_OFFSET_MASK;