6289479 libresolv2 clean up and alignment with libbind.6.0

1 files changed, 50 insertions, 69 deletions

diff --git a/usr/src/lib/libresolv2/common/dst/hmac_link.c b/usr/src/lib/libresolv2/common/dst/hmac_link.c
index 77269667c2..071e8d2aad 100644
--- a/usr/src/lib/libresolv2/common/dst/hmac_link.c
+++ b/usr/src/lib/libresolv2/common/dst/hmac_link.c
@@ -1,13 +1,12 @@
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
 
 #ifdef HMAC_MD5
 #ifndef LINT
-static const char rcsid[] = "$Header: /proj/cvs/isc/bind8/src/lib/dst/hmac_link.c,v 1.9 2001/05/29 05:48:10 marka Exp $";
+static const char rcsid[] = "$Header: /proj/cvs/prod/libbind/dst/hmac_link.c,v 1.8 2007/09/24 17:18:25 each Exp $";
 #endif
 /*
  * Portions Copyright (c) 1995-1998 by Trusted Information Systems, Inc.
@@ -26,7 +25,7 @@ static const char rcsid[] = "$Header: /proj/cvs/isc/bind8/src/lib/dst/hmac_link.
  * WITH THE USE OR PERFORMANCE OF THE SOFTWARE.
  */
 
-/* 
+/*%
  * This file contains an implementation of the HMAC-MD5 algorithm.
  */
 #include "port_before.h"
@@ -43,14 +42,17 @@ static const char rcsid[] = "$Header: /proj/cvs/isc/bind8/src/lib/dst/hmac_link.
 #include <resolv.h>
 
 #include "dst_internal.h"
+
 #ifdef USE_MD5
-#ifndef	SUNW_LIBMD5
-# include "md5.h"
-#else
-#include <sys/md5.h>
-#endif
+# ifndef HAVE_MD5
+#  include "md5.h"
+# else
+#  ifdef SOLARIS2
+#   include <sys/md5.h>
+#  endif
+# endif
 # ifndef _MD5_H_
-#  define _MD5_H_ 1	/* make sure we do not include rsaref md5.h file */
+#  define _MD5_H_ 1	/*%< make sure we do not include rsaref md5.h file */
 # endif
 #endif
 
@@ -97,6 +99,9 @@ dst_hmac_md5_sign(const int mode, DST_KEY *d_key, void **context,
 	int sign_len = 0;
 	MD5_CTX *ctx = NULL;
 
+	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
+		return (-1);
+
 	if (mode & SIG_MODE_INIT) 
 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
 	else if (context)
@@ -104,8 +109,6 @@ dst_hmac_md5_sign(const int mode, DST_KEY *d_key, void **context,
 	if (ctx == NULL) 
 		return (-1);
 
-	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
-		return (-1);
 	key = (HMAC_Key *) d_key->dk_KEY_struct;
 
 	if (mode & SIG_MODE_INIT) {
@@ -164,6 +167,9 @@ dst_hmac_md5_verify(const int mode, DST_KEY *d_key, void **context,
 	HMAC_Key *key;
 	MD5_CTX *ctx = NULL;
 
+	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
+		return (-1);
+
 	if (mode & SIG_MODE_INIT) 
 		ctx = (MD5_CTX *) malloc(sizeof(*ctx));
 	else if (context)
@@ -171,9 +177,6 @@ dst_hmac_md5_verify(const int mode, DST_KEY *d_key, void **context,
 	if (ctx == NULL) 
 		return (-1);
 
-	if (d_key == NULL || d_key->dk_KEY_struct == NULL)
-		return (-1);
-
 	key = (HMAC_Key *) d_key->dk_KEY_struct;
 	if (mode & SIG_MODE_INIT) {
 		MD5Init(ctx);
@@ -226,6 +229,7 @@ dst_buffer_to_hmac_md5(DST_KEY *dkey, const u_char *key, const int keylen)
 	HMAC_Key *hkey = NULL;
 	MD5_CTX ctx;
 	int local_keylen = keylen;
+	u_char tk[MD5_LEN];
 
 	if (dkey == NULL || key == NULL || keylen < 0)
 		return (-1);
@@ -238,7 +242,6 @@ dst_buffer_to_hmac_md5(DST_KEY *dkey, const u_char *key, const int keylen)
 
 	/* if key is longer than HMAC_LEN bytes reset it to key=MD5(key) */
 	if (keylen > HMAC_LEN) {
-		u_char tk[MD5_LEN];
 		MD5Init(&ctx);
 		MD5Update(&ctx, key, keylen);
 		MD5Final(tk, &ctx);
@@ -276,25 +279,28 @@ dst_buffer_to_hmac_md5(DST_KEY *dkey, const u_char *key, const int keylen)
 
 static int
 dst_hmac_md5_key_to_file_format(const DST_KEY *dkey, char *buff,
-			    const int buff_len)
+			        const int buff_len)
 {
 	char *bp;
-	int len, b_len, i, key_len;
+	int len, i, key_len;
 	u_char key[HMAC_LEN];
 	HMAC_Key *hkey;
 
 	if (dkey == NULL || dkey->dk_KEY_struct == NULL) 
 		return (0);
-	if (buff == NULL || buff_len <= (int) strlen(key_file_fmt_str))
-		return (-1);	/* no OR not enough space in output area */
-
+	/*
+	 * Using snprintf() would be so much simpler here.
+	 */
+	if (buff == NULL ||
+	    buff_len <= (int)(strlen(key_file_fmt_str) +
+			      strlen(KEY_FILE_FORMAT) + 4))
+		return (-1);	/*%< no OR not enough space in output area */
 	hkey = (HMAC_Key *) dkey->dk_KEY_struct;
-	memset(buff, 0, buff_len);	/* just in case */
+	memset(buff, 0, buff_len);	/*%< just in case */
 	/* write file header */
 	sprintf(buff, key_file_fmt_str, KEY_FILE_FORMAT, KEY_HMAC_MD5, "HMAC");
 
-	bp = (char *) strchr(buff, '\0');
-	b_len = buff_len - (bp - buff);
+	bp = buff + strlen(buff);
 
 	memset(key, 0, HMAC_LEN);
 	for (i = 0; i < HMAC_LEN; i++)
@@ -304,19 +310,21 @@ dst_hmac_md5_key_to_file_format(const DST_KEY *dkey, char *buff,
 			break;
 	key_len = i + 1;
 
+	if (buff_len - (bp - buff) < 6)
+		return (-1);
 	strcat(bp, "Key: ");
 	bp += strlen("Key: ");
-	b_len = buff_len - (bp - buff);
 
-	len = b64_ntop(key, key_len, bp, b_len);
+	len = b64_ntop(key, key_len, bp, buff_len - (bp - buff));
 	if (len < 0) 
 		return (-1);
 	bp += len;
+	if (buff_len - (bp - buff) < 2)
+		return (-1);
 	*(bp++) = '\n';
 	*bp = '\0';
-	b_len = buff_len - (bp - buff);
 
-	return (buff_len - b_len);
+	return (bp - buff);
 }
 
 
@@ -338,9 +346,9 @@ dst_hmac_md5_key_from_file_format(DST_KEY *dkey, const char *buff,
 {
 	const char *p = buff, *eol;
 	u_char key[HMAC_LEN+1];	/* b64_pton needs more than 64 bytes do decode
-							 * it should probably be fixed rather than doing
-							 * this
-							 */
+				 * it should probably be fixed rather than doing
+				 * this
+				 */
 	u_char *tmp;
 	int key_len, len;
 
@@ -359,9 +367,11 @@ dst_hmac_md5_key_from_file_format(DST_KEY *dkey, const char *buff,
 		return (-4);
 	len = eol - p;
 	tmp = malloc(len + 2);
+	if (tmp == NULL)
+		return (-5);
 	memcpy(tmp, p, len);
 	*(tmp + len) = 0x0;
-	key_len = b64_pton((char *)tmp, key, HMAC_LEN+1);	/* see above */
+	key_len = b64_pton((char *)tmp, key, HMAC_LEN+1);	/*%< see above */
 	SAFE_FREE2(tmp, len + 2);
 
 	if (dst_buffer_to_hmac_md5(dkey, key, key_len) < 0) {
@@ -370,7 +380,7 @@ dst_hmac_md5_key_from_file_format(DST_KEY *dkey, const char *buff,
 	return (0);
 }
 
-/*
+/*%
  * dst_hmac_md5_to_dns_key() 
  *         function to extract hmac key from DST_KEY structure 
  * intput: 
@@ -439,43 +449,17 @@ dst_hmac_md5_free_key_structure(void *key)
 static int
 dst_hmac_md5_generate_key(DST_KEY *key, const int nothing)
 {
-	u_char *buff;
-	int i, n, size;
-
-	i = nothing;
-
-	if (key == NULL || key->dk_alg != KEY_HMAC_MD5)
-		return (0);
-	size = (key->dk_key_size + 7) / 8; /* convert to bytes */
-	if (size <= 0)
-		return(0);
-	
-	i = size > 64 ? 64 : size;
-	buff = malloc(i+8);
-
-	n = dst_random(DST_RAND_SEMI, i, buff);
-	n += dst_random(DST_RAND_KEY, i, buff);
-	if (n <= i) {	/* failed getting anything */
-		SAFE_FREE2(buff, i);
-		return (-1);
-	}
-	n = dst_buffer_to_hmac_md5(key, buff, i);
-	SAFE_FREE2(buff, i);
-	if (n <= 0)
-		return (n);
-	return (1);
+	(void)key;
+	(void)nothing;
+	return (-1);
 }
 
-/*
+/*%
  * dst_hmac_md5_init()  Function to answer set up function pointers for HMAC
  *	   related functions 
  */
 int
-#ifdef	ORIGINAL_ISC_CODE
 dst_hmac_md5_init()
-#else
-dst_md5_hmac_init()
-#endif
 {
 	if (dst_t_func[KEY_HMAC_MD5] != NULL)
 		return (1);
@@ -496,15 +480,12 @@ dst_md5_hmac_init()
 }
 
 #else 
+#define	dst_hmac_md5_init	__dst_hmac_md5_init
+
 int
 dst_hmac_md5_init(){
 	return (0);
 }
 #endif
 
-
-
-
-
-
-
+/*! \file */