6907 strcoll() and strxfrm() don't seem to agree

Reviewed by: Robert Mustacchi <rm@joyent.com>
Approved by: Dan McDonald <danmcd@omniti.com>

1 files changed, 61 insertions, 59 deletions

diff --git a/usr/src/lib/libc/port/locale/wcscoll.c b/usr/src/lib/libc/port/locale/wcscoll.c
index 26b39c9da2..4813362c6c 100644
--- a/usr/src/lib/libc/port/locale/wcscoll.c
+++ b/usr/src/lib/libc/port/locale/wcscoll.c
@@ -1,6 +1,6 @@
 /*
  * Copyright 2013 Garrett D'Amore <garrett@damore.org>
- * Copyright 2010 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2017 Nexenta Systems, Inc.
  * Copyright (c) 2002 Tim J. Robbins
  * All rights reserved.
  *
@@ -38,19 +38,19 @@
 int
 wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 {
-	int len1, len2, pri1, pri2, ret;
+	int len1, len2, pri1, pri2;
 	wchar_t *tr1 = NULL, *tr2 = NULL;
 	int direc, pass;
 	const struct lc_collate *lcc = loc->collate;
+	int ret = wcscmp(ws1, ws2);
 
-	if (lcc->lc_is_posix)
-		/*
-		 * Locale has no special collating order or could not be
-		 * loaded, do a fast binary comparison.
-		 */
-		return (wcscmp(ws1, ws2));
+	if (lcc->lc_is_posix || ret == 0)
+		return (ret);
 
-	ret = 0;
+	if (*ws1 == 0 && *ws2 != 0)
+		return (-1);
+	if (*ws1 != 0 && *ws2 == 0)
+		return (1);
 
 	/*
 	 * Once upon a time we had code to try to optimize this, but
@@ -65,24 +65,23 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 	 * up UNDEFINED elements.  There is special handling for them.
 	 */
 	for (pass = 0; pass <= lcc->lc_directive_count; pass++) {
-
 		const int32_t *st1 = NULL;
 		const int32_t *st2 = NULL;
 		const wchar_t	*w1 = ws1;
 		const wchar_t	*w2 = ws2;
-		int check1, check2;
 
 		/* special pass for UNDEFINED */
 		if (pass == lcc->lc_directive_count) {
-			direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
+			direc = DIRECTIVE_FORWARD;
 		} else {
 			direc = lcc->lc_directive[pass];
 		}
 
 		if (direc & DIRECTIVE_BACKWARD) {
 			wchar_t *bp, *fp, c;
+			free(tr1);
 			if ((tr1 = wcsdup(w1)) == NULL)
-				goto fail;
+				goto end;
 			bp = tr1;
 			fp = tr1 + wcslen(tr1) - 1;
 			while (bp < fp) {
@@ -90,8 +89,9 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 				*bp++ = *fp;
 				*fp-- = c;
 			}
+			free(tr2);
 			if ((tr2 = wcsdup(w2)) == NULL)
-				goto fail;
+				goto end;
 			bp = tr2;
 			fp = tr2 + wcslen(tr2) - 1;
 			while (bp < fp) {
@@ -104,6 +104,7 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 		}
 
 		if (direc & DIRECTIVE_POSITION) {
+			int check1, check2;
 			while (*w1 && *w2) {
 				pri1 = pri2 = 0;
 				check1 = check2 = 1;
@@ -113,7 +114,7 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 						    &pri1, pass, &st1);
 						if (pri1 < 0) {
 							errno = EINVAL;
-							goto fail;
+							goto end;
 						}
 						if (!pri1) {
 							/*CSTYLED*/
@@ -127,7 +128,7 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 						    &pri2, pass, &st2);
 						if (pri2 < 0) {
 							errno = EINVAL;
-							goto fail;
+							goto end;
 						}
 						if (!pri2) {
 							/*CSTYLED*/
@@ -144,58 +145,63 @@ wcscoll_l(const wchar_t *ws1, const wchar_t *ws2, locale_t loc)
 				w1 += len1;
 				w2 += len2;
 			}
+			if (!*w1) {
+				if (*w2) {
+					ret = -(int)*w2;
+					goto end;
+				}
+			} else {
+				ret = *w1;
+				goto end;
+			}
 		} else {
-			while (*w1 && *w2) {
-				pri1 = pri2 = 0;
-				check1 = check2 = 1;
-				while ((pri1 == pri2) && (check1 || check2)) {
-					while (check1 && *w1) {
-						_collate_lookup(lcc, w1, &len1,
-						    &pri1, pass, &st1);
-						if (pri1 > 0)
-							break;
-						if (pri1 < 0) {
-							errno = EINVAL;
-							goto fail;
-						}
-						st1 = NULL;
-						w1 += 1;
+			int vpri1 = 0, vpri2 = 0;
+			while (*w1 || *w2 || st1 || st2) {
+				pri1 = 1;
+				while (*w1 || st1) {
+					_collate_lookup(lcc, w1, &len1, &pri1,
+					    pass, &st1);
+					w1 += len1;
+					if (pri1 > 0) {
+						vpri1++;
+						break;
 					}
-					check1 = (st1 != NULL);
-					while (check2 && *w2) {
-						_collate_lookup(lcc, w2, &len2,
-						    &pri2, pass, &st2);
-						if (pri2 > 0)
-							break;
-						if (pri2 < 0) {
-							errno = EINVAL;
-							goto fail;
-						}
-						st2 = NULL;
-						w2 += 1;
+					if (pri1 < 0) {
+						errno = EINVAL;
+						goto end;
 					}
-					check2 = (st2 != NULL);
-					if (!pri1 || !pri2)
+					st1 = NULL;
+				}
+				pri2 = 1;
+				while (*w2 || st2) {
+					_collate_lookup(lcc, w2, &len2, &pri2,
+					    pass, &st2);
+					w2 += len2;
+					if (pri2 > 0) {
+						vpri2++;
 						break;
+					}
+					if (pri2 < 0) {
+						errno = EINVAL;
+						goto end;
+					}
+					st2 = NULL;
 				}
-				if (!pri1 || !pri2)
+				if ((!pri1 || !pri2) && (vpri1 == vpri2))
 					break;
 				if (pri1 != pri2) {
 					ret = pri1 - pri2;
 					goto end;
 				}
-				w1 += len1;
-				w2 += len2;
 			}
-		}
-		if (!*w1) {
-			if (*w2) {
-				ret = -(int)*w2;
+			if (vpri1 && !vpri2) {
+				ret = 1;
+				goto end;
+			}
+			if (!vpri1 && vpri2) {
+				ret = -1;
 				goto end;
 			}
-		} else {
-			ret = *w1;
-			goto end;
 		}
 	}
 	ret = 0;
@@ -205,10 +211,6 @@ end:
 	free(tr2);
 
 	return (ret);
-
-fail:
-	ret = wcscmp(ws1, ws2);
-	goto end;
 }
 
 int