15036 portfs wears inadequate pollcache disguise

Reviewed by: Andy Fiddaman <illumos@fiddaman.net>
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Dan McDonald <danmcd@mnx.io>

4 files changed, 30 insertions, 8 deletions

diff --git a/usr/src/uts/common/fs/portfs/port.c b/usr/src/uts/common/fs/portfs/port.c
index dd32c82434..c7e069b108 100644
--- a/usr/src/uts/common/fs/portfs/port.c
+++ b/usr/src/uts/common/fs/portfs/port.c
@@ -26,6 +26,7 @@
 
 /*
  * Copyright (c) 2015 Joyent, Inc.  All rights reserved.
+ * Copyright 2022 Oxide Computer Company
  */
 
 #include <sys/types.h>
@@ -824,6 +825,7 @@ port_init(port_t *pp)
 	/* Allocate cache skeleton for PORT_SOURCE_FD events */
 	portq->portq_pcp = kmem_zalloc(sizeof (port_fdcache_t), KM_SLEEP);
 	mutex_init(&portq->portq_pcp->pc_lock, NULL, MUTEX_DEFAULT, NULL);
+	portq->portq_pcp->pc_flag = PC_PORTFS;
 
 	/*
 	 * Allocate cache skeleton for association of event sources.
diff --git a/usr/src/uts/common/sys/poll_impl.h b/usr/src/uts/common/sys/poll_impl.h
index 5ecf7433aa..ff277f89c8 100644
--- a/usr/src/uts/common/sys/poll_impl.h
+++ b/usr/src/uts/common/sys/poll_impl.h
@@ -233,9 +233,11 @@ struct polldat {
 /*
  * One cache for each thread that polls. Points to a bitmap (used by pollwakeup)
  * and a hash table of polldats.
- * The offset of pc_lock field must be kept in sync with the pc_lock offset
- * of port_fdcache_t, both structs implement pc_lock with offset 0 (see also
- * pollrelock()).
+ *
+ * Because of the handling required in pollrelock(), portfs abuses the notion of
+ * an active pollcache (t_pollcache), providing its own struct port_fdcache_t.
+ * It has matching pc_lock and pc_flag members at the correct offsets, but none
+ * of its other fields can be accessed (through t_pollcache) safetly.
  */
 struct pollcache {
 	kmutex_t	pc_lock;	/* lock to protect pollcache */
@@ -259,6 +261,13 @@ struct pollcache {
 /* pc_flag */
 #define	PC_POLLWAKE	0x02	/* pollwakeup() occurred */
 #define	PC_EPOLL	0x04	/* pollcache is epoll-enabled */
+/*
+ * PC_PORTFS is not a flag for "real" pollcaches, but rather an indicator for
+ * when portfs sets t_pollcache to a port_fdcache_t pointer.  If, while
+ * debugging a system, one sees PC_PORTFS in pc_flag, they will know to
+ * disregard the other fields, as it is not a pollcache.
+ */
+#define	PC_PORTFS	0x08
 
 #if defined(_KERNEL)
 /*
diff --git a/usr/src/uts/common/sys/port_kernel.h b/usr/src/uts/common/sys/port_kernel.h
index 7456f63573..daac6f4927 100644
--- a/usr/src/uts/common/sys/port_kernel.h
+++ b/usr/src/uts/common/sys/port_kernel.h
@@ -22,13 +22,12 @@
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2022 Oxide Computer Company
  */
 
 #ifndef	_SYS_PORT_KERNEL_H
 #define	_SYS_PORT_KERNEL_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/vnode.h>
 #include <sys/list.h>
 
@@ -52,7 +51,7 @@ extern "C" {
 typedef	struct	port_kevent {
 	kmutex_t	portkev_lock;	/* used by PORT_SOURCE_FD source */
 	int	portkev_source;		/* event: source */
-	int	portkev_events; 	/* event: data */
+	int	portkev_events;		/* event: data */
 	int	portkev_flags;		/* internal flags */
 	pid_t	portkev_pid;		/* pid of process using this struct */
 	long	portkev_object;		/* event: object */
@@ -122,8 +121,10 @@ typedef struct portfop_cache {
 /*
  * PORT_SOURCE_FD cache per port.
  * One cache for each port that uses PORT_SOURCE_FD.
- * pc_lock must be the first element of port_fdcache_t to keep it
- * synchronized with the offset of pc_lock in pollcache_t (see pollrelock()).
+ *
+ * The types and offsets of pc_lock and pc_flag must exactly match their sibling
+ * fields in pollcache_t, as they are accessed as if the port_fdcache_t _was_ a
+ * pollcache via t_pollcache. (See: pollrelock() and fs_reject_epoll())
  */
 typedef struct port_fdcache {
 	kmutex_t	pc_lock;	/* lock to protect portcache */
@@ -131,6 +132,8 @@ typedef struct port_fdcache {
 	struct portfd	**pc_hash;	/* points to a hash table of ptrs */
 	int		pc_hashsize;	/* the size of current hash table */
 	int		pc_fdcount;	/* track how many fd's are hashed */
+	uintptr_t	_pc_pad;	/* pad to properly offset pc_flag */
+	int		pc_flag;	/* pollcache flags (compat) */
 } port_fdcache_t;
 
 /*
diff --git a/usr/src/uts/common/syscall/poll.c b/usr/src/uts/common/syscall/poll.c
index 71069314d7..7af1c7edfe 100644
--- a/usr/src/uts/common/syscall/poll.c
+++ b/usr/src/uts/common/syscall/poll.c
@@ -279,6 +279,14 @@ pollunlock(int *lockstate)
 	return (0);
 }
 
+/*
+ * The pc_lock and pc_flag fields of port_fdcache_t must exactly match those of
+ * pollcache_t as they are accessed through t_pollcache as if they were part of
+ * a "real" pollcache.
+ */
+CTASSERT(offsetof(pollcache_t, pc_lock) == offsetof(port_fdcache_t, pc_lock));
+CTASSERT(offsetof(pollcache_t, pc_flag) == offsetof(port_fdcache_t, pc_flag));
+
 void
 pollrelock(int lockstate)
 {