PSARC 2006/162 Extended FILE space for 32-bit Solaris processes

1085341 32-bit stdio routines should support file descriptors >255
6369408 fflush(NULL); will corrupt data written on files in multithreaded apps

3 files changed, 33 insertions, 18 deletions

diff --git a/usr/src/ucblib/libucb/inc/stdiom.h b/usr/src/ucblib/libucb/inc/stdiom.h
index 0ec6e4940b..0cd4d59114 100644
--- a/usr/src/ucblib/libucb/inc/stdiom.h
+++ b/usr/src/ucblib/libucb/inc/stdiom.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 1997 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -98,6 +97,12 @@ extern int	_xflsbuf(FILE *iop);
 	 */
 extern Uchar 	*_findbuf(FILE *iop);
 
+#ifndef _LP64
+extern int _file_set(FILE *, int, const char *);
+#define	SET_FILE(iop, fd)	(iop)->_magic = (fd); (iop)->__extendedfd = 0
+#define	_FILE_FD_MAX		255
+#endif
+
 /*
  * The following macros improve performance of the stdio by reducing the
  * number of calls to _bufsync and _wrtchk.  _needsync checks whether
diff --git a/usr/src/ucblib/libucb/port/gen/setbuffer.c b/usr/src/ucblib/libucb/port/gen/setbuffer.c
index 65983dca4b..47aa76f88b 100644
--- a/usr/src/ucblib/libucb/port/gen/setbuffer.c
+++ b/usr/src/ucblib/libucb/port/gen/setbuffer.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,11 +18,6 @@
  *
  * CDDL HEADER END
  */
-/*
- * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
- */
-
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
 /*	  All Rights Reserved  	*/
 
@@ -37,6 +31,11 @@
  * contributors.
  */
 
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
 #pragma ident	"%Z%%M%	%I%	%E% SMI"
 
 /*LINTLIBRARY*/
@@ -53,7 +52,7 @@ void
 setbuffer(FILE *iop, char *abuf, int asize)
 {
 	Uchar *buf = (Uchar *)abuf;
-	int fno = iop->_file;  /* file number */
+	int fno = fileno(iop);  /* file number */
 	int size = asize - _SMBFSZ;
 	Uchar *temp;
 
diff --git a/usr/src/ucblib/libucb/port/stdio/fopen.c b/usr/src/ucblib/libucb/port/stdio/fopen.c
index edfa76228e..c6fcb4ee42 100644
--- a/usr/src/ucblib/libucb/port/stdio/fopen.c
+++ b/usr/src/ucblib/libucb/port/stdio/fopen.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 1998 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -74,6 +74,7 @@ _endopen(const char *file, const char *mode, FILE *iop, int largefile)
 	default:
 		return (NULL);
 	}
+
 	if (largefile) {
 		fd = open64(file, oflag, 0666);	/* mapped to open() for V9 */
 	} else {
@@ -82,7 +83,17 @@ _endopen(const char *file, const char *mode, FILE *iop, int largefile)
 	if (fd < 0)
 		return (NULL);
 	iop->_cnt = 0;
-	iop->_file = (unsigned char) fd;
+#ifdef _LP64
+	iop->_file = fd;
+#else
+	if (fd <= _FILE_FD_MAX) {
+		SET_FILE(iop, fd);
+	} else if (_file_set(iop, fd, mode) != 0) {
+		/* errno set in _file_set() */
+		(void) close(fd);
+		return (NULL);
+	}
+#endif
 	iop->_flag = plus ? _IORW : (mode[0] == 'r') ? _IOREAD : _IOWRT;
 	if (mode[0] == 'a')   {
 		if ((lseek64(fd, 0L, SEEK_END)) < 0)  {