6833994 LDoms domain services module needs to support more than 64 ports

3 files changed, 52 insertions, 26 deletions

diff --git a/usr/src/uts/sun4v/io/ds_common.c b/usr/src/uts/sun4v/io/ds_common.c
index aa2d847bda..8ad1dc65d3 100644
--- a/usr/src/uts/sun4v/io/ds_common.c
+++ b/usr/src/uts/sun4v/io/ds_common.c
@@ -59,6 +59,7 @@
  */
 ds_port_t	ds_ports[DS_MAX_PORTS];
 ds_portset_t	ds_allports;	/* all DS ports in the system */
+ds_portset_t	ds_nullport;	/* allows test against null portset */
 
 /*
  * Table of registered services
@@ -1237,8 +1238,17 @@ ds_try_next_port(ds_svc_t *svc, int portid)
 	int i;
 
 	DS_DBG_LDC(CE_NOTE, "ds@%x %s" DS_EOL, portid, __func__);
-	DS_PORTSET_NOT(totry, svc->tried);
-	DS_PORTSET_AND(totry, svc->avail);
+
+	/*
+	 * Get the ports that haven't been tried yet and are available to try.
+	 */
+	DS_PORTSET_SETNULL(totry);
+	for (i = 0; i < DS_MAX_PORTS; i++) {
+		if (!DS_PORT_IN_SET(svc->tried, i) &&
+		    DS_PORT_IN_SET(svc->avail, i))
+			DS_PORTSET_ADD(totry, i);
+	}
+
 	if (DS_PORTSET_ISNULL(totry))
 		return;
 
@@ -2097,7 +2107,7 @@ ds_svc_register(ds_svc_t *svc, void *arg)
 	if (DS_SVC_ISFREE(svc))
 		return (0);
 
-	ports = svc->avail;
+	DS_PORTSET_DUP(ports, svc->avail);
 	if (svc->flags & DSSF_ISCLIENT) {
 		ds_portset_del_active_clients(svc->cap.svc_id, &ports);
 	} else if (svc->state != DS_SVC_INACTIVE)
@@ -3139,13 +3149,14 @@ ds_hdl_lookup(char *service, uint_t is_client, ds_svc_hdl_t *hdlp,
 static void
 ds_portset_del_active_clients(char *service, ds_portset_t *portsp)
 {
-	ds_portset_t ports = *portsp;
+	ds_portset_t ports;
 	int idx;
 	ds_svc_t *svc;
 	ds_svc_hdl_t hdl;
 
 	ASSERT(MUTEX_HELD(&ds_svcs.lock));
 
+	DS_PORTSET_DUP(ports, *portsp);
 	for (idx = 0; idx < ds_svcs.maxsvcs; idx++) {
 		svc = ds_svcs.tbl[idx];
 		if (DS_SVC_ISFREE(svc))
@@ -3169,7 +3180,7 @@ ds_portset_del_active_clients(char *service, ds_portset_t *portsp)
 	    svc->port != NULL) {
 		DS_PORTSET_DEL(ports, PORTID(svc->port));
 	}
-	*portsp = ports;
+	DS_PORTSET_DUP(*portsp, ports);
 }
 
 /*
diff --git a/usr/src/uts/sun4v/sys/ds_impl.h b/usr/src/uts/sun4v/sys/ds_impl.h
index 718c01416c..9aae6b51fa 100644
--- a/usr/src/uts/sun4v/sys/ds_impl.h
+++ b/usr/src/uts/sun4v/sys/ds_impl.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -31,6 +31,8 @@
 extern "C" {
 #endif
 
+#include <sys/bitmap.h>
+#include <sys/ldoms.h>
 
 
 /*
@@ -204,25 +206,28 @@ typedef struct ds_port {
 
 /*
  * A DS portset is a bitmap that represents a collection of DS
- * ports. Each bit represent a particular port id. The current
- * implementation constrains the maximum number of ports to 64.
+ * ports. Each bit represent a particular port id.  We need
+ * to allocate for the max. number of domains supported,
+ * plus a small number (e.g. for the SP connection).
  */
-typedef uint64_t ds_portset_t;
+#define	DS_EXTRA_PORTS			16
+#define	DS_MAX_PORTS			(LDOMS_MAX_DOMAINS + DS_EXTRA_PORTS)
+#define	DS_PORTSET_SIZE			BT_BITOUL(DS_MAX_PORTS)
+
+typedef ulong_t ds_portset_t[DS_PORTSET_SIZE];
+
+extern ds_portset_t ds_nullport;
 
-#ifndef	DS_MAX_PORTS
-#define	DS_MAX_PORTS			((sizeof (ds_portset_t)) * 8)
-#endif
 #define	DS_MAX_PORT_ID			(DS_MAX_PORTS - 1)
 
-#define	DS_PORT_SET(port)		(1UL << (port))
-#define	DS_PORT_IN_SET(set, port)	((set) & DS_PORT_SET(port))
-#define	DS_PORTSET_ADD(set, port)	((void)((set) |= DS_PORT_SET(port)))
-#define	DS_PORTSET_DEL(set, port)	((void)((set) &= ~DS_PORT_SET(port)))
-#define	DS_PORTSET_ISNULL(set)		((set) == 0)
-#define	DS_PORTSET_SETNULL(set)		((void)((set) = 0))
-#define	DS_PORTSET_DUP(set1, set2)	((void)((set1) = (set2)))
-#define	DS_PORTSET_NOT(set1, set2)	((void)((set1) = ~(set2)))
-#define	DS_PORTSET_AND(set1, set2)	((void)((set1) &= (set2)))
+#define	DS_PORT_IN_SET(set, port)	BT_TEST((set), (port))
+#define	DS_PORTSET_ADD(set, port)	BT_SET((set), (port))
+#define	DS_PORTSET_DEL(set, port)	BT_CLEAR((set), (port))
+#define	DS_PORTSET_ISNULL(set)		(memcmp((set), ds_nullport, \
+					    sizeof (set)) == 0)
+#define	DS_PORTSET_SETNULL(set)		((void)memset((set), 0, sizeof (set)))
+#define	DS_PORTSET_DUP(set1, set2)	((void)memcpy((set1), (set2), \
+					    sizeof (set1)))
 
 /*
  * A DS event consists of a buffer on a port.  We explictly use a link to
diff --git a/usr/src/uts/sun4v/sys/ldoms.h b/usr/src/uts/sun4v/sys/ldoms.h
index e01f15893d..09c28f828a 100644
--- a/usr/src/uts/sun4v/sys/ldoms.h
+++ b/usr/src/uts/sun4v/sys/ldoms.h
@@ -20,15 +20,13 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_LDOMS_H
 #define	_LDOMS_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -39,8 +37,20 @@ extern "C" {
  * Global LDoms definitions.
  */
 
-/* Maximum number of logical domains supported */
-#define	LDOMS_MAX_DOMAINS	32
+/*
+ * LDOMS_MAX_DOMAINS refers to the maximum theoretical number of
+ * domains supported by Solaris based on per-domain resources that
+ * are allocated. The actual number of domains supported by a
+ * platform is defined by the firmware.
+ *
+ * When adjusting this value please ensure that resources such
+ * as the following are approriately scaled:
+ *    - channel nexus interrupt cookies
+ *    - domain services ports
+ *    - NCPUS
+ *    - etc...
+ */
+#define	LDOMS_MAX_DOMAINS	512
 
 /* maximum number of characters in the logical domain name */
 #define	LDOMS_MAX_NAME_LEN	MAXHOSTNAMELEN