path: root/usr/src/cmd/sort/common/fields.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include "fields.h"

/*
 * fields
 *
 * Overview
 *   By a field, we mean the various delimited character sequences within each
 *   line of the input files.  The sort key consists of an ordered sequence of
 *   fields, which need not include all possible fields for the given line.
 *   (Furthermore, not every line need contain sufficient fields for the fields
 *   given within the sort key.  In fact, none of the lines in the input stream
 *   need contain sufficient fields.)
 *
 *   There are two methods for specifying fields for sort(1); these are
 *   discussed in options.c.  Here we discuss only the internal representation
 *   of fields, as used for constructing the collation vector for each line as
 *   defined by the sort key.
 *
 * Representation
 *   The sort key is a singly-linked list of field specifiers.  At present,
 *   fields may belong to one of three species:  alphabetical, numerical, or
 *   monthly; the species (f_species) then indicates the conversion function
 *   (f_convert) used to transform the raw characters of the character sequence
 *   to a collatable form.  (In principle, this allows us to consider future
 *   field species such as hexadecimal.)
 *
 *   Fields and offsets are numbered such that zero refers to the first field or
 *   character, respectively.  Thus, the interpretation of a key specifier, m.n,
 *   is that the field begins at the nth character beyond the mth occurence of
 *   the key separator.  If the blanks flag has been specified, then the field
 *   begins at the nth non-blank character past the mth key separator.  If the
 *   key separator is unspecified, then the key separator is defined as one or
 *   more blank characters.
 *
 *   In general, the various options afforded by sort may be broken into two
 *   categories:  field species and field modifiers.  For each field species,
 *   there is one or more conversion routines that take a delimited character
 *   sequence and convert it to a character sequence collatable by strcmp() or
 *   memcmp().  For field species that may be further modified, such as the
 *   fold-to-uppercase option for alphabetic fields, the conversion routine may
 *   be aware of how the modifier affects collation.  Finally, the no-modifiers
 *   case may present an opportunity for a simplified, faster version.
 *
 * Code Structure
 *   The code paths for single-byte and multi-byte locales diverge significantly
 *   in fields.c.  Most routines have an *_wide() version, which produces an
 *   equivalent effect for line records whose data field is composed of wide
 *   characters (wchar_t).  However, the l_collated field of a line record is
 *   always composed of characters, so that the radix sorts provided in
 *   internal.c can work in both single- and multi-byte locales.  Thus, in the
 *   various convert_*_wide() routines, the output is placed in l_collated, with
 *   a length multiplier of 4.
 */

#define	BEFORE_NUMBER	0x0
#define	IN_NUMBER	0x1

static char	numerical_separator;
static char	numerical_decimal;
static char	monetary_separator;
static char	monetary_decimal;

static wchar_t	w_numerical_separator;
static wchar_t	w_numerical_decimal;
static wchar_t	w_monetary_separator;
static wchar_t	w_monetary_decimal;

#define	MONTHS_IN_YEAR	12
#define	MAX_MON_LEN	20

enum { MO_NONE = 1, MO_OFFSET = 2 };

static char	*months[MONTHS_IN_YEAR];
static size_t	month_lengths[MONTHS_IN_YEAR];
static wchar_t	*w_months[MONTHS_IN_YEAR];
static size_t	w_month_lengths[MONTHS_IN_YEAR];

#define	DECIMAL_CHAR		(numerical_decimal)
#define	IS_BLANK(x)		(isspace((uchar_t)(x)) && (x) != '\n')
#define	IS_SEPARATOR(x)		\
	((numerical_separator != '\0' && (x) == numerical_separator) || \
	(monetary_separator != '\0' && (x) == monetary_separator))
#define	IS_DECIMAL(x)		\
	((x) == numerical_decimal || \
	(monetary_decimal != '\0' && (x) == monetary_decimal))
#define	W_DECIMAL_CHAR		(w_numerical_decimal)
#define	W_IS_BLANK(x)		(iswspace(x) && (x) != L'\n')
#define	W_IS_SEPARATOR(x)	\
	((numerical_separator != '\0' && (x) == w_numerical_separator) || \
	(monetary_separator != '\0' && (x) == w_monetary_separator))
#define	W_IS_DECIMAL(x)		\
	(((x) == w_numerical_decimal) || \
	(monetary_decimal != '\0' && (x) == w_monetary_decimal))

#define	INTERFIELD_SEPARATOR '\0'
#define	W_INTERFIELD_SEPARATOR L'\0'

#define	INT_SIGN_FLIP_MASK 0x80000000
#define	INT_SIGN_PASS_MASK 0x00000000

/*
 * strx_ops_t, xfrm_len, and xfrm_cpy:  In the case where we are sorting in the
 * C locale, we want to avoid the expense of transforming strings to collatable
 * forms since, by definition, an arbitrary string in the C locale is already in
 * its collatable form.  Therefore, we construct a small ops vector (the
 * strx_ops) and two wrappers: xfrm_len() to massage the strxfrm(NULL, ...) into
 * strlen()-like behaviour, and xfrm_cpy() to make strncpy() appear
 * strxfrm()-like.
 */
/*ARGSUSED*/
static size_t
xfrm_len(const char *s2, size_t len)
{
	return (strxfrm(NULL, s2, 0) + 1);
}

/*
 * The length represented by n includes a null character, so to return the
 * correct length we subtract 1.  Note that this function is only used by
 * field_convert_alpha, and isn't for general use, as it assumes that n is the
 * length of s2 plus a null character.
 */
static size_t
C_ncpy(char *s1, const char *s2, size_t n)
{
	(void) strncpy(s1, s2, n);
	return (n - 1);
}

/*ARGSUSED*/
static size_t
C_len(const char *s, size_t len)
{
	ASSERT(s != NULL);
	return (len);
}

typedef struct _strx_ops {
	size_t	(*sx_len)(const char *, size_t);
	size_t	(*sx_xfrm)(char *, const char *, size_t);
} strx_ops_t;

static const strx_ops_t C_ops = { C_len, C_ncpy };
static const strx_ops_t SB_ops = { xfrm_len, strxfrm };

static const strx_ops_t *xfrm_ops;

static void
field_initialize_separator(void)
{
	/*
	 * A locale need not define all of the cases below:  only decimal_point
	 * must be defined.  Furthermore, sort(1) has traditionally not used the
	 * positive_sign and negative_sign, grouping, or currency_symbols (or
	 * their numeric counterparts, if any).
	 */
	struct lconv *conv = localeconv();

	if (!xstreql(conv->thousands_sep, "")) {
		numerical_separator = *conv->thousands_sep;
		(void) mbtowc(&w_numerical_separator, conv->thousands_sep,
		    MB_CUR_MAX);
	} else
		numerical_separator = '\0';

	if (!xstreql(conv->mon_thousands_sep, "")) {
		monetary_separator = *conv->mon_thousands_sep;
		(void) mbtowc(&w_monetary_separator, conv->mon_thousands_sep,
		    MB_CUR_MAX);
	} else
		monetary_separator = '\0';

	if (!xstreql(conv->mon_decimal_point, "")) {
		monetary_decimal = *conv->mon_decimal_point;
		(void) mbtowc(&w_monetary_decimal, conv->mon_decimal_point,
		    MB_CUR_MAX);
	} else
		monetary_decimal = '\0';

	numerical_decimal = *conv->decimal_point;
	(void) mbtowc(&w_numerical_decimal, conv->decimal_point, MB_CUR_MAX);
}

static void
field_initialize_month(int is_c_locale)
{
	int i;
	int j;
	struct tm this_month;
	const char *c_months[MONTHS_IN_YEAR] = {
		"JAN", "FEB", "MAR", "APR", "MAY", "JUN",
		"JUL", "AUG", "SEP", "OCT", "NOV", "DEC"
	};

	char month_name[MAX_MON_LEN * MB_LEN_MAX];
	wchar_t	w_month_name[MAX_MON_LEN];

	if (is_c_locale) {
		for (i = 0; i < MONTHS_IN_YEAR; i++) {
			months[i] = (char *)c_months[i];
			month_lengths[i] = strlen(c_months[i]);
		}
		/*
		 * We don't need to initialize the wide version of the month
		 * names.
		 */
		return;
	}

	(void) memset(&this_month, 0, sizeof (this_month));

	for (i = 0; i < MONTHS_IN_YEAR; i++) {
		this_month.tm_mon = i;

		(void) strftime(month_name, sizeof (month_name),
		    "%b", &this_month);

		for (j = 0; j < strlen(month_name); j++)
			month_name[j] = toupper(month_name[j]);
		(void) mbstowcs(w_month_name, month_name, MAX_MON_LEN);

		months[i] = strdup(month_name);
		month_lengths[i] = strlen(month_name);
		w_months[i] = wsdup(w_month_name);
		w_month_lengths[i] = wslen(w_month_name);
	}
}

void
field_initialize(sort_t *S)
{
	field_initialize_month(S->m_c_locale);
	field_initialize_separator();

	if (S->m_c_locale)
		xfrm_ops = &C_ops;
	else
		xfrm_ops = &SB_ops;
}

field_t *
field_new(sort_t *S)
{
	field_t	*F = safe_realloc(NULL, sizeof (field_t));

	F->f_start_field = -1;
	F->f_start_offset = -1;
	F->f_end_field = -1;
	F->f_end_offset = -1;
	F->f_next = NULL;

	if (S == NULL) {
		F->f_species = ALPHA;
		F->f_options = 0;
	} else {
		F->f_species = S->m_default_species;
		F->f_options = S->m_field_options;
	}

	return (F);
}

void
field_delete(field_t *F)
{
	free(F);
}

/*
 * The recursive implementation of field_add_to_chain() given below is
 * inappropriate if function calls are expensive, or a truly large number of
 * fields are anticipated.
 */
void
field_add_to_chain(field_t **F, field_t *A)
{
	if (*F == NULL)
		*F = A;
	else
		field_add_to_chain(&((*F)->f_next), A);
}

#ifdef DEBUG
#ifndef _LP64
#define	FIELD_FMT \
"\nStart field: %d\tStart offset: %d\nEnd field: %d\tEnd offset: %d\n"
#else /* !_LP64 */
#define	FIELD_FMT \
"\nStart field: %ld\tStart offset: %ld\nEnd field: %ld\tEnd offset: %ld\n"
#endif /* !_LP64 */

/*
 * field_print is used only for debugging purposes.
 */
void
field_print(field_t *F)
{
	char *field_names[] = {"ALPHA", "MONTH", "NUMERIC"};
	int status = 0;

	(void) fprintf(stderr, "Type: %s", field_names[F->f_species]);
	(void) fprintf(stderr, "\tOptions: ");

	if (F->f_options & FIELD_REVERSE_COMPARISONS) {
		(void) fprintf(stderr, "REVERSE");
		status++;
	}
	if (F->f_options & FIELD_DICTIONARY_ORDER) {
		(void) fprintf(stderr, "DICTIONARY ");
		status++;
	}
	if (F->f_options & FIELD_FOLD_UPPERCASE) {
		(void) fprintf(stderr, "UPPERCASE ");
		status++;
	}
	if (F->f_options & FIELD_IGNORE_NONPRINTABLES) {
		(void) fprintf(stderr, "PRINTABLES ");
		status++;
	}
	if (F->f_options & FIELD_IGNORE_BLANKS_START) {
		(void) fprintf(stderr, "BLANKS_START ");
		status++;
	}
	if (F->f_options & FIELD_IGNORE_BLANKS_END) {
		(void) fprintf(stderr, "BLANKS_END ");
		status++;
	}

	if (status == 0)
		(void) fprintf(stderr, "NO_MODIFIERS");

	(void) fprintf(stderr, FIELD_FMT, F->f_start_field, F->f_start_offset,
	    F->f_end_field, F->f_end_offset);
}
#endif /* DEBUG */

static ssize_t
field_boundary(field_t *F, line_rec_t *L, int is_end, int is_blanks)
{
	char *S = L->l_data.sp;
	char *T = S;
	char *eol = S + L->l_data_length;
	ssize_t field = is_end ? F->f_end_field : F->f_start_field;
	ssize_t offset = is_end ? F->f_end_offset : F->f_start_offset;
	ssize_t ret;

	ASSERT(is_end || field > -1);

	if (is_end && field == -1)
		return (L->l_data_length);

	while (field-- > 0) {
		while (T < eol && IS_BLANK(*T))
			T++;

		while (T < eol && !IS_BLANK(*T))
			T++;
	}

	if ((!is_end || offset > 0) && is_blanks) {
		while (IS_BLANK(*T))
			T++;
	}

	if ((ret = MAX(T - S, 0) + offset) >= L->l_data_length)
		return (L->l_data_length);

	return (ret);
}

static void
field_delimit(field_t *F, line_rec_t *L, ssize_t *start, ssize_t *end)
{
	ASSERT(F->f_start_field > -1);

	*start = field_boundary(F, L, 0,
	    F->f_options & FIELD_IGNORE_BLANKS_START);
	*end = field_boundary(F, L, 1,
	    F->f_options & FIELD_IGNORE_BLANKS_END);
}

static ssize_t
field_boundary_wide(field_t *F, line_rec_t *L, int is_end, int is_blanks)
{
	wchar_t *S = L->l_data.wp;
	wchar_t *T = S;
	wchar_t *eol = S + L->l_data_length;
	ssize_t field = is_end ? F->f_end_field : F->f_start_field;
	ssize_t offset = is_end ? F->f_end_offset : F->f_start_offset;
	ssize_t ret;

	ASSERT(is_end || field > -1);

	if (is_end && field == -1)
		return (L->l_data_length);

	while (field-- > 0) {
		while (T < eol && W_IS_BLANK(*T))
			T++;

		while (T < eol && !W_IS_BLANK(*T))
			T++;
	}

	if ((!is_end || offset > 0) && is_blanks) {
		while (W_IS_BLANK(*T))
			T++;
	}

	if ((ret = MAX(T - S, 0) + offset) >= L->l_data_length)
		return (L->l_data_length);

	return (ret);
}

static void
field_delimit_wide(field_t *F, line_rec_t *L, ssize_t *start, ssize_t *end)
{
	ASSERT(F->f_start_field > -1);

	*start = field_boundary_wide(F, L, 0,
	    F->f_options & FIELD_IGNORE_BLANKS_START);
	*end = field_boundary_wide(F, L, 1,
	    F->f_options & FIELD_IGNORE_BLANKS_END);
}

static ssize_t
field_boundary_tabbed(field_t *F, line_rec_t *L, int is_end, int is_blanks,
    vchar_t delimiter)
{
	char *S = L->l_data.sp;
	char *T = S;
	char *eol = S + L->l_data_length;
	ssize_t field = is_end ? F->f_end_field : F->f_start_field;
	ssize_t offset = is_end ? F->f_end_offset : F->f_start_offset;
	ssize_t ret;

	ASSERT(is_end || field > -1);

	if (is_end && field == -1)
		return (L->l_data_length);

	while (field-- > 0) {
		T = xstrnchr(T, delimiter.sc, eol - T);
		if (T == NULL || T > eol)
			return (L->l_data_length);

		T++;
	}

	if ((!is_end || offset != 0) && is_blanks) {
		while (IS_BLANK(*T))
			T++;
	}

	if ((ret = MAX(T - S, 0) + offset) >= L->l_data_length) {
		if (L->l_data_length <= 0)
			return (0);
		if (S[L->l_data_length - 1] == delimiter.sc) {
			return (L->l_data_length - 1);
		} else {
			return (L->l_data_length);
		}
	}

	if (is_end && offset == 0)
		ret--;

	return (ret);
}

/*
 * field_delimit_tabbed() is called when a field separator has been defined
 * using the -t option.  The character at the offset, start, is either one or
 * more character positions past the delimiter marking the start of the
 * field, or at the end of the line.
 */
static void
field_delimit_tabbed(field_t *F, line_rec_t *L, ssize_t *start, ssize_t *end,
    vchar_t delimiter)
{
	ASSERT(F->f_start_field > -1);

	*start = field_boundary_tabbed(F, L, 0, F->f_options &
	    FIELD_IGNORE_BLANKS_START, delimiter);
	*end = field_boundary_tabbed(F, L, 1, F->f_options &
	    FIELD_IGNORE_BLANKS_END, delimiter);
}

static ssize_t
field_boundary_tabbed_wide(field_t *F, line_rec_t *L, int is_end, int is_blanks,
    vchar_t delimiter)
{
	wchar_t *S = L->l_data.wp;
	wchar_t *T = S;
	wchar_t *eol = S + L->l_data_length;
	ssize_t field = is_end ? F->f_end_field : F->f_start_field;
	ssize_t offset = is_end ? F->f_end_offset : F->f_start_offset;
	ssize_t ret;

	ASSERT(is_end || field > -1);

	if (is_end && field == -1)
		return (L->l_data_length);

	while (field-- > 0) {
		T = xwsnchr(T, delimiter.wc, eol - T);
		if (T == NULL || T > eol)
			return (L->l_data_length);

		T++;
	}

	if ((!is_end || offset != 0) && is_blanks) {
		while (W_IS_BLANK(*T))
			T++;
	}

	if ((ret = MAX(T - S, 0) + offset) >= L->l_data_length) {
		if (L->l_data_length <= 0)
			return (0);
		if (S[L->l_data_length - 1] == delimiter.wc) {
			return (L->l_data_length - 1);
		} else {
			return (L->l_data_length);
		}
	}

	if (is_end && offset == 0)
		ret--;

	return (ret);
}

static void
field_delimit_tabbed_wide(field_t *F, line_rec_t *L, ssize_t *start,
    ssize_t *end, vchar_t delimiter)
{
	ASSERT(F->f_start_field > -1);

	*start = field_boundary_tabbed_wide(F, L, 0, F->f_options &
	    FIELD_IGNORE_BLANKS_START, delimiter);
	*end = field_boundary_tabbed_wide(F, L, 1, F->f_options &
	    FIELD_IGNORE_BLANKS_END, delimiter);
}

/*ARGSUSED*/
ssize_t
field_convert_month(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	int j;
	ssize_t	val;
	char month_candidate[MAX_MON_LEN * MB_LEN_MAX];
	ssize_t month_length = data_length;
	ssize_t month_offset = data_offset;

	if (sizeof (char) > L->l_collate_bufsize - coll_offset)
		return (-1);

	(void) memset(month_candidate, 0, MAX_MON_LEN * MB_LEN_MAX);


	/*
	 * The month field formally begins with the first non-blank character.
	 */
	while (IS_BLANK(*(L->l_data.sp + month_offset))) {
		month_offset++;
		month_length--;
	}

	for (j = 0; j < MAX_MON_LEN && j < month_length; j++)
		month_candidate[j] = toupper((L->l_data.sp + month_offset)[j]);

	for (j = 0; j < MONTHS_IN_YEAR; j++) {
		if (xstrneql(month_candidate, months[j], month_lengths[j])) {
			*(L->l_collate.sp + coll_offset) = '\0' + j + MO_OFFSET;
			return (1);
		}
	}

	/*
	 * no matching month; copy string into field.  required behaviour is
	 * that "month-free" keys sort before month-sortable keys, so insert
	 * a "will sort first" token.
	 */
	*(L->l_collate.sp + coll_offset) = '\0' + MO_NONE;

	val = field_convert_alpha_simple(F, L, delimiter, data_offset,
	    data_length, coll_offset + 1);

	if (val < 0)
		return (-1);
	else
		return (val + 1);
}

/*ARGSUSED*/
ssize_t
field_convert_month_wide(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	ssize_t j;
	ssize_t val;
	wchar_t month_candidate[MAX_MON_LEN];
	wchar_t *month;
	wchar_t *buffer = L->l_collate.wp + coll_offset;
	ssize_t month_length = data_length;
	ssize_t month_offset = data_offset;

	if (L->l_collate_bufsize - coll_offset * sizeof (wchar_t) <
	    sizeof (wchar_t))
		return (-1);

	(void) memset(month_candidate, 0, MAX_MON_LEN * sizeof (wchar_t));


	while (W_IS_BLANK(*(L->l_data.wp + month_offset))) {
		month_offset++;
		month_length--;
	}

	month = L->l_data.wp + month_offset;

	for (j = 0; j < MAX_MON_LEN && j < month_length; j++)
		month_candidate[j] = towupper(month[j]);

	for (j = 0; j < MONTHS_IN_YEAR; j++)
		if (xwcsneql(month_candidate, w_months[j],
		    w_month_lengths[j])) {
			*buffer = L'\0' + j + MO_OFFSET;
			return (1);
		}

	*buffer = L'\0' + MO_NONE;

	val = field_convert_alpha_wide(F, L, delimiter, data_offset,
	    data_length, coll_offset + sizeof (wchar_t));

	if (val < 0)
		return (-1);
	else
		return (val + 1);
}

/*
 * field_convert_alpha() always fails with return value -1 if the converted
 * string would cause l_collate_length to exceed l_collate_bufsize
 */
/*ARGSUSED*/
ssize_t
field_convert_alpha(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	static char *compose;
	static ssize_t compose_length;

	ssize_t	clength = 0;
	ssize_t	dlength;
	ssize_t	i;

	if (compose_length < (data_length + 1)) {
		compose_length = data_length + 1;
		compose = safe_realloc(compose, compose_length * sizeof (char));
	}

	for (i = data_offset; i < data_offset + data_length; i++) {
		char t = (L->l_data.sp)[i];

		if ((F->f_options & FIELD_IGNORE_NONPRINTABLES) &&
		    !isprint((uchar_t)t))
			continue;

		if ((F->f_options & FIELD_DICTIONARY_ORDER) &&
		    !isalnum((uchar_t)t) && !isspace((uchar_t)t))
			continue;

		if (F->f_options & FIELD_FOLD_UPPERCASE)
			t = toupper(t);

		compose[clength++] = t;
	}
	compose[clength] = '\0';

	if ((dlength = xfrm_ops->sx_len(compose, clength)) <
	    L->l_collate_bufsize - coll_offset)
		return (xfrm_ops->sx_xfrm(L->l_collate.sp + coll_offset,
		    compose, dlength + 1));
	else
		return ((ssize_t)-1);
}

/*ARGSUSED*/
ssize_t
field_convert_alpha_simple(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	static char *compose;
	static ssize_t compose_length;

	ssize_t	clength;
	ssize_t	dlength;

	if (compose_length < (data_length + 1)) {
		compose_length = data_length + 1;
		compose = safe_realloc(compose, compose_length * sizeof (char));
	}

	(void) memcpy(compose, L->l_data.sp + data_offset, data_length);
	clength = data_length;
	compose[clength] = '\0';

	if ((dlength = xfrm_ops->sx_len(compose, clength)) <
	    L->l_collate_bufsize - coll_offset)
		return (xfrm_ops->sx_xfrm(L->l_collate.sp + coll_offset,
		    compose, dlength + 1));
	else
		return ((ssize_t)-1);
}

/*ARGSUSED*/
ssize_t
field_convert_alpha_wide(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	wchar_t	*compose = safe_realloc(NULL, (data_length + 1) *
	    sizeof (wchar_t));
	ssize_t	clength = 0;
	ssize_t	dlength;
	ssize_t	i;
	ssize_t ret;

	for (i = data_offset; i < data_offset + data_length; i++) {
		wchar_t	t = (L->l_data.wp)[i];

		if ((F->f_options & FIELD_IGNORE_NONPRINTABLES) && !iswprint(t))
			continue;

		if ((F->f_options & FIELD_DICTIONARY_ORDER) && !iswalnum(t) &&
		    !iswspace(t))
			continue;

		if (F->f_options & FIELD_FOLD_UPPERCASE)
			t = towupper(t);

		compose[clength++] = t;
	}
	compose[clength] = L'\0';

	dlength = wcsxfrm(NULL, compose, (size_t)0);
	if ((dlength * sizeof (wchar_t)) < L->l_collate_bufsize -
	    coll_offset * sizeof (wchar_t)) {
		ret = (ssize_t)wcsxfrm(L->l_collate.wp + coll_offset, compose,
		    (size_t)dlength + 1);
	} else {
		ret = (ssize_t)-1;
	}

	safe_free(compose);

	return (ret);
}

/*
 * field_convert_numeric() converts the given field into a collatable numerical
 * sequence.  The sequence is ordered as { log, integer, separator, fraction },
 * with an optional sentinel component at the sequence end.
 */
/*ARGSUSED*/
ssize_t
field_convert_numeric(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	char *number;
	char *buffer = L->l_collate.sp + coll_offset;
	ssize_t length;

	char sign = '2';
	int log_ten;
	char *digits = buffer + 1 + sizeof (int) / sizeof (char);
	size_t j = 0;
	size_t i;

	int state = BEFORE_NUMBER;

	number = L->l_data.sp + data_offset;
	length = data_length;

	/*
	 * Eat leading blanks, if any.
	 */
	for (i = 0; i < length; i++)
		if (!IS_BLANK(number[i]))
			break;

	/*
	 * Test that there is sufficient size in the collation buffer for our
	 * number.  In addition to the possible remaining characters in the
	 * field, we also require space for the sign (char), logarithm (int),
	 * separator (char), and as many as two string terminators (for reverse
	 * sorts).
	 */
	if (((length - i) + 4 * sizeof (char) + sizeof (int)) >
	    (L->l_collate_bufsize - coll_offset))
		return ((ssize_t)-1);

	/*
	 * If negative, set sign.
	 */
	if (number[i] == '-') {
		i++;
		sign = '0';
	}

	/*
	 * Scan integer part; eat leading zeros.
	 */
	for (; i < length; i++) {
		if (IS_SEPARATOR(number[i]))
			continue;

		if (number[i] == '0' && !(state & IN_NUMBER))
			continue;

		if (!isdigit((uchar_t)number[i]))
			break;

		state |= IN_NUMBER;
		if (sign == '0')
			digits[j++] = '0' + '9' - number[i];
		else
			digits[j++] = number[i];
	}

	if (i < length && IS_DECIMAL(number[i])) {
		/*
		 * Integer part terminated by decimal.
		 */
		digits[j] = DECIMAL_CHAR;
		log_ten = j++;

		/*
		 * Scan fractional part.
		 */
		for (++i; i < length; i++) {
			if (IS_SEPARATOR(number[i]))
				continue;

			if (!isdigit((uchar_t)number[i]))
				break;

			if (number[i] != '0')
				state |= IN_NUMBER;

			if (sign == '0')
				digits[j++] = '0' + '9' - number[i];
			else
				digits[j++] = number[i];
		}

		if (sign == '0')
			digits[j++] = (char)(UCHAR_MAX - INTERFIELD_SEPARATOR);
	} else {
		/*
		 * Nondigit or end of string seen.
		 */
		log_ten = (int)j;
		if (sign == '0')
			digits[j++] = (char)(UCHAR_MAX - INTERFIELD_SEPARATOR);
		else
			digits[j] = INTERFIELD_SEPARATOR;
	}

	if ((state & IN_NUMBER) == 0) {
		/*
		 * A non-zero number was not detected; treat as defined zero.
		 */
		sign = '1';
		log_ten = 0;
		digits[0] = '0';
		j = 1;
	}

	/*
	 * We subtract a constant from the log of negative values so that
	 * they will correctly precede positive values with a zero logarithm.
	 */
	if (sign == '0') {
		if (j != 0)
			log_ten = -log_ten - 2;
		else
			/*
			 * Special case for -0.
			 */
			log_ten = -1;
	}

	buffer[0] = sign;

	/*
	 * Place logarithm in big-endian form.
	 */
	for (i = 0; i < sizeof (int); i++)
		buffer[i + 1] = (log_ten << (i * NBBY))
		    >> ((sizeof (int) - 1) * NBBY);

	if (j + sizeof (char) + sizeof (int) <
	    L->l_collate_bufsize - coll_offset)
		return (j + 1 + sizeof (int));
	else
		return ((ssize_t)-1);
}

/*ARGSUSED*/
ssize_t
field_convert_numeric_wide(field_t *F, line_rec_t *L, vchar_t delimiter,
    ssize_t data_offset, ssize_t data_length, ssize_t coll_offset)
{
	wchar_t *number;
	wchar_t *buffer = L->l_collate.wp + coll_offset;
	char *lbuffer;
	ssize_t length;

	wchar_t	sign = L'2';
	int log_ten;
	wchar_t	*digits = buffer + 1 + sizeof (int)/sizeof (wchar_t);
	size_t j = 0;
	size_t i;

	int state = BEFORE_NUMBER;

	number = L->l_data.wp + data_offset;
	length = data_length;

	for (i = 0; i < length; i++)
		if (!W_IS_BLANK(number[i]))
			break;

	if (((length - i) * sizeof (wchar_t) + 4 * sizeof (wchar_t) +
	    sizeof (int)) > (L->l_collate_bufsize - coll_offset))
		return ((ssize_t)-1);

	if (number[i] == L'-') {
		i++;
		sign = L'0';
	}

	for (; i < length; i++) {
		if (W_IS_SEPARATOR(number[i]))
			continue;

		if (number[i] == L'0' && !(state & IN_NUMBER))
			continue;

		if (!iswdigit(number[i]))
			break;

		state |= IN_NUMBER;
		if (sign == L'0')
			digits[j++] = L'0' + L'9' - number[i];
		else
			digits[j++] = number[i];
	}

	if (i < length && W_IS_DECIMAL(number[i])) {
		digits[j] = W_DECIMAL_CHAR;
		log_ten = j++;

		for (++i; i < length; i++) {
			if (W_IS_SEPARATOR(number[i]))
				continue;

			if (!iswdigit(number[i]))
				break;

			if (number[i] != L'0')
				state |= IN_NUMBER;

			if (sign == L'0')
				digits[j++] = L'0' + L'9' - number[i];
			else
				digits[j++] = number[i];
		}

		if (sign == L'0')
			digits[j++] = (wchar_t)(WCHAR_MAX -
			    W_INTERFIELD_SEPARATOR);
	} else {
		log_ten = (int)j;
		if (sign == L'0')
			digits[j++] = (wchar_t)(WCHAR_MAX -
			    W_INTERFIELD_SEPARATOR);
		else
			digits[j] = W_INTERFIELD_SEPARATOR;
	}

	if ((state & IN_NUMBER) == 0) {
		sign = L'1';
		log_ten = 0;
		digits[0] = L'0';
		j = 1;
	}

	if (sign == L'0') {
		if (j != 0)
			log_ten = -log_ten - 2;
		else
			log_ten = -1;
	}

	buffer[0] = sign;
	/*
	 * Place logarithm in big-endian form.
	 */
	lbuffer = (char *)(buffer + 1);
	for (i = 0; i < sizeof (int); i++)
		lbuffer[i] = (log_ten << (i * NBBY))
		    >> ((sizeof (int) - 1) * NBBY);

	if ((j + 1 + sizeof (int)/sizeof (wchar_t)) * sizeof (wchar_t) <
	    L->l_collate_bufsize - coll_offset * sizeof (wchar_t))
		return (j + 1 + sizeof (int) / sizeof (wchar_t));
	else
		return ((ssize_t)-1);
}

/*
 * flags contains one of CV_REALLOC, CV_FAIL, specifying the preferred behaviour
 * when coll_offset exceeds l_collate_bufsize.
 */
ssize_t
field_convert(field_t *F, line_rec_t *L, int flags, vchar_t field_separator)
{
	ssize_t coll_offset = 0;
	ssize_t	start, end, distance;
	field_t *cur_fieldp = F;

	while (cur_fieldp != NULL) {
		/*
		 * delimit field
		 */
		if (!field_separator.sc)
			field_delimit(cur_fieldp, L, &start, &end);
		else
			field_delimit_tabbed(cur_fieldp, L, &start, &end,
			    field_separator);

		distance = 0;
		if (end - start > 0 ||
		    (end - start == 0 && F->f_species == NUMERIC)) {
			/*
			 * Convert field, appending to collated field of line
			 * record.
			 */
			distance = cur_fieldp->f_convert(cur_fieldp, L,
			    field_separator, start, end - start, coll_offset);

			/*
			 * branch should execute comparatively rarely
			 */
			if (distance == -1) {
				if (flags & FCV_REALLOC) {
					ASSERT(L->l_collate_bufsize > 0);
					L->l_collate_bufsize *= 2;
					L->l_collate.sp =
					    safe_realloc(L->l_collate.sp,
					    L->l_collate_bufsize);

					__S(stats_incr_convert_reallocs());
					continue;
				} else {
					/*
					 * FCV_FAIL has been set.
					 */
					return (-1);
				}
			}
		}

		if (cur_fieldp->f_options & FIELD_REVERSE_COMPARISONS) {
			xstrninv(L->l_collate.sp, coll_offset, distance);
			*(L->l_collate.sp + coll_offset + distance) =
			    (char)(UCHAR_MAX - INTERFIELD_SEPARATOR);
			distance++;
		}

		ASSERT(distance >= 0);
		coll_offset += distance;
		if (coll_offset >= L->l_collate_bufsize) {
			if (flags & FCV_REALLOC) {
				ASSERT(L->l_collate_bufsize > 0);
				L->l_collate_bufsize *= 2;
				L->l_collate.sp = safe_realloc(L->l_collate.sp,
				    L->l_collate_bufsize);

				__S(stats_incr_convert_reallocs());
			} else {
				return (-1);
			}
		}
		*(L->l_collate.sp + coll_offset) = INTERFIELD_SEPARATOR;
		coll_offset++;

		cur_fieldp = cur_fieldp->f_next;
	}

	L->l_collate_length = coll_offset;

	return (L->l_collate_length);
}

ssize_t
field_convert_wide(field_t *F, line_rec_t *L, int flags,
    vchar_t field_separator)
{
	ssize_t coll_offset = 0;
	ssize_t	start, end, distance;
	field_t *cur_fieldp = F;

	while (cur_fieldp != NULL) {
		if (!field_separator.wc)
			field_delimit_wide(cur_fieldp, L, &start, &end);
		else
			field_delimit_tabbed_wide(cur_fieldp, L, &start, &end,
			    field_separator);

		distance = 0;
		if (end - start > 0 ||
		    end - start == 0 && F->f_species == NUMERIC) {
			distance = cur_fieldp->f_convert(cur_fieldp, L,
			    field_separator, start, end - start, coll_offset);

			if (distance == -1) {
				if (flags & FCV_REALLOC) {
					ASSERT(L->l_collate_bufsize > 0);
					L->l_collate_bufsize *= 2;
					L->l_collate.wp = safe_realloc(
					    L->l_collate.wp,
					    L->l_collate_bufsize);

					__S(stats_incr_convert_reallocs());
					continue;
				} else {
					return (-1);
				}
			}
		}

		if (cur_fieldp->f_options & FIELD_REVERSE_COMPARISONS) {
			xwcsninv(L->l_collate.wp, coll_offset, distance);
			*(L->l_collate.wp + coll_offset + distance) =
			    WCHAR_MAX - INTERFIELD_SEPARATOR;
			distance++;
		}

		ASSERT(distance >= 0);
		coll_offset += distance;
		if (coll_offset * sizeof (wchar_t) >= L->l_collate_bufsize) {
			if (flags & FCV_REALLOC) {
				ASSERT(L->l_collate_bufsize > 0);
				L->l_collate_bufsize *= 2;
				L->l_collate.wp = safe_realloc(L->l_collate.wp,
				    L->l_collate_bufsize);

				__S(stats_incr_convert_reallocs());
			} else {
				return (-1);
			}
		}
		*(L->l_collate.wp + coll_offset) = W_INTERFIELD_SEPARATOR;
		coll_offset++;

		cur_fieldp = cur_fieldp->f_next;
	}

	L->l_collate_length = coll_offset * sizeof (wchar_t);
#ifdef _LITTLE_ENDIAN
	xwcsntomsb(L->l_collate.wp, coll_offset);
#endif /* _LITTLE_ENDIAN */

	return (L->l_collate_length);
}

/*
 * line_convert() and line_convert_wide() are called when the collation vector
 * of a given line has been exhausted, and we are performing the final,
 * full-line comparison required by the sort specification.  Because we do not
 * have a guarantee that l_data is null-terminated, we create an explicitly
 * null-terminated copy suitable for transformation to a collatable form for the
 * current locale.
 */
static void
line_convert(line_rec_t *L)
{
	static ssize_t bufsize;
	static char *buffer;

	if (L->l_raw_collate.sp != NULL)
		return;

	if (L->l_data_length + 1 > bufsize) {
		buffer = safe_realloc(buffer, L->l_data_length + 1);
		bufsize = L->l_data_length + 1;
	}

	(void) strncpy(buffer, L->l_data.sp, L->l_data_length);
	buffer[L->l_data_length] = '\0';

	L->l_raw_collate.sp = safe_realloc(L->l_raw_collate.sp,
	    xfrm_ops->sx_len(buffer, L->l_data_length) + 1);
	xfrm_ops->sx_xfrm(L->l_raw_collate.sp, buffer,
	    xfrm_ops->sx_len(buffer, L->l_data_length) + 1);

	__S(stats_incr_line_conversions());
}

static void
line_convert_wide(line_rec_t *L)
{
	static wchar_t *buffer;
	static ssize_t bufsize;

	ssize_t dlength;

	if (L->l_raw_collate.wp != NULL)
		return;

	if (L->l_data_length + 1 > bufsize) {
		buffer = safe_realloc(buffer, (L->l_data_length + 1) *
		    sizeof (wchar_t));
		bufsize = L->l_data_length + 1;
	}

	(void) wcsncpy(buffer, L->l_data.wp, L->l_data_length);
	buffer[L->l_data_length] = L'\0';

	dlength = wcsxfrm(NULL, buffer, 0) + 1;
	L->l_raw_collate.wp = safe_realloc(L->l_raw_collate.wp, dlength *
	    sizeof (wchar_t));
	(void) wcsxfrm(L->l_raw_collate.wp, buffer, dlength);

	__S(stats_incr_line_conversions());
}

/*
 * Our convention for collation is
 *
 *	A > B  => r > 0,
 *	A == B => r = 0,
 *	A < B  => r < 0
 *
 * This convention is consistent with the definition of memcmp(), strcmp(), and
 * strncmp() in the C locale.  collated() and collated_wide() have two optional
 * behaviours, which can be activated by setting the appropriate values in
 * coll_flag:  COLL_UNIQUE, which returns 0 if the l_collate fields of the line
 * records being compared are identical; COLL_DATA_ONLY, which ignores the
 * l_collate field for the current comparison; and COLL_REVERSE, which flips the
 * result for comparisons that fall through to an actual data comparison (since
 * the collated vector should already reflect reverse ordering from field
 * conversion).
 */
int
collated(line_rec_t *A, line_rec_t *B, ssize_t depth, flag_t coll_flag)
{
	ssize_t ml = MIN(A->l_collate_length, B->l_collate_length) - depth;
	int r;
	int mask = (coll_flag & COLL_REVERSE) ? INT_SIGN_FLIP_MASK :
	    INT_SIGN_PASS_MASK;
	ssize_t la, lb;

	if (!(coll_flag & COLL_DATA_ONLY)) {
		if (ml > 0) {
			r = memcmp(A->l_collate.sp + depth,
			    B->l_collate.sp + depth, ml);

			if (r)
				return (r);
		}

		if (A->l_collate_length < B->l_collate_length)
			return (-1);

		if (A->l_collate_length > B->l_collate_length)
			return (1);
	}

	/*
	 * This is where we cut out, if we know that the current sort is over
	 * the entire line.
	 */
	if (coll_flag & COLL_UNIQUE)
		return (0);

	line_convert(A);
	line_convert(B);

	la = strlen(A->l_raw_collate.sp);
	lb = strlen(B->l_raw_collate.sp);

	r = memcmp(A->l_raw_collate.sp, B->l_raw_collate.sp, MIN(la, lb));

	if (r)
		return (r ^ mask);

	if (la < lb)
		return (-1 ^ mask);

	if (la > lb)
		return (1 ^ mask);

	return (0);
}

int
collated_wide(line_rec_t *A, line_rec_t *B, ssize_t depth, flag_t coll_flag)
{
	ssize_t ml = MIN(A->l_collate_length, B->l_collate_length) - depth;
	int r;
	int mask = (coll_flag & COLL_REVERSE) ? INT_SIGN_FLIP_MASK :
	    INT_SIGN_PASS_MASK;
	ssize_t la, lb;

	if (!(coll_flag & COLL_DATA_ONLY)) {
		if (ml > 0) {
			r = memcmp(A->l_collate.sp + depth,
			    B->l_collate.sp + depth, ml);

			if (r)
				return (r);
		}
		if (A->l_collate_length < B->l_collate_length)
			return (-1);

		if (A->l_collate_length > B->l_collate_length)
			return (1);
	}

	if (coll_flag & COLL_UNIQUE)
		return (0);

	line_convert_wide(A);
	line_convert_wide(B);

	la = wcslen(A->l_raw_collate.wp);
	lb = wcslen(B->l_raw_collate.wp);

	r = wmemcmp(A->l_raw_collate.wp, B->l_raw_collate.wp,
	    (size_t)MIN(la, lb));

	if (r)
		return (r ^ mask);

	if (la < lb)
		return (-1 ^ mask);

	if (la > lb)
		return (1 ^ mask);

	return (0);
}