OS-5189 lx dev enumeration can deadlock with zfs

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>

5 files changed, 33 insertions, 19 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 672eba850e..1643eaf8a2 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -176,7 +176,7 @@
 
 int	lx_debug = 0;
 
-void	lx_init_brand_data(zone_t *);
+void	lx_init_brand_data(zone_t *, kmutex_t *);
 void	lx_free_brand_data(zone_t *);
 void	lx_setbrand(proc_t *);
 int	lx_getattr(zone_t *, int, void *, size_t *);
@@ -995,9 +995,10 @@ lx_zfs_cleanup_devs(lx_zone_data_t *lxzdata)
 }
 
 void
-lx_init_brand_data(zone_t *zone)
+lx_init_brand_data(zone_t *zone, kmutex_t *zsl)
 {
 	lx_zone_data_t *data;
+	ASSERT(MUTEX_HELD(zsl));
 	ASSERT(zone->zone_brand == &lx_brand);
 	ASSERT(zone->zone_brand_data == NULL);
 	data = (lx_zone_data_t *)kmem_zalloc(sizeof (lx_zone_data_t), KM_SLEEP);
@@ -1015,6 +1016,14 @@ lx_init_brand_data(zone_t *zone)
 	(void) strlcpy(data->lxzd_kernel_version, "BrandZ virtual linux",
 	    LX_KERN_VERSION_MAX);
 
+	zone->zone_brand_data = data;
+
+	/*
+	 * In Linux, if the init(1) process terminates the system panics.
+	 * The zone must reboot to simulate this behaviour.
+	 */
+	zone->zone_reboot_on_init_exit = B_TRUE;
+
 	/*
 	 * Unlike ZFS proper, which does dynamic zvols, we currently only
 	 * generate the zone's "disk" list once at zone boot time and use that
@@ -1029,15 +1038,15 @@ lx_init_brand_data(zone_t *zone)
 	list_create(data->lxzd_vdisks, sizeof (lxd_zfs_dev_t),
 	    offsetof(lxd_zfs_dev_t, lzd_link));
 
-	lx_zfs_get_devs(zone, data->lxzd_vdisks);
-
-	zone->zone_brand_data = data;
-
 	/*
-	 * In Linux, if the init(1) process terminates the system panics.
-	 * The zone must reboot to simulate this behaviour.
+	 * We cannot hold the zone_status_lock while performing zfs operations
+	 * so we drop the lock, get the zfs devs as the last step in this
+	 * function, then reaquire the lock. Don't add any code after this
+	 * which requires that the zone_status_lock was continuously held.
 	 */
-	zone->zone_reboot_on_init_exit = B_TRUE;
+	mutex_exit(zsl);
+	lx_zfs_get_devs(zone, data->lxzd_vdisks);
+	mutex_enter(zsl);
 }
 
 void
diff --git a/usr/src/uts/common/brand/sn1/sn1_brand.c b/usr/src/uts/common/brand/sn1/sn1_brand.c
index eb99142327..dc68c7c0a7 100644
--- a/usr/src/uts/common/brand/sn1/sn1_brand.c
+++ b/usr/src/uts/common/brand/sn1/sn1_brand.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2015 Joyent, Inc. All rights reserved.
+ * Copyright 2016 Joyent, Inc.
  */
 
 #include <sys/errno.h>
@@ -43,7 +43,7 @@
 
 char *sn1_emulation_table = NULL;
 
-void	sn1_init_brand_data(zone_t *);
+void	sn1_init_brand_data(zone_t *, kmutex_t *);
 void	sn1_free_brand_data(zone_t *);
 void	sn1_setbrand(proc_t *);
 int	sn1_getattr(zone_t *, int, void *, size_t *);
@@ -244,7 +244,7 @@ sn1_free_brand_data(zone_t *zone)
 
 /*ARGSUSED*/
 void
-sn1_init_brand_data(zone_t *zone)
+sn1_init_brand_data(zone_t *zone, kmutex_t *zsl)
 {
 }
 
diff --git a/usr/src/uts/common/brand/solaris10/s10_brand.c b/usr/src/uts/common/brand/solaris10/s10_brand.c
index 345d2151b3..39f56d6495 100644
--- a/usr/src/uts/common/brand/solaris10/s10_brand.c
+++ b/usr/src/uts/common/brand/solaris10/s10_brand.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, Joyent, Inc. All rights reserved.
+ * Copyright 2016, Joyent, Inc.
  */
 
 #include <sys/errno.h>
@@ -46,7 +46,7 @@
 
 char *s10_emulation_table = NULL;
 
-void	s10_init_brand_data(zone_t *);
+void	s10_init_brand_data(zone_t *, kmutex_t *);
 void	s10_free_brand_data(zone_t *);
 void	s10_setbrand(proc_t *);
 int	s10_getattr(zone_t *, int, void *, size_t *);
@@ -410,7 +410,7 @@ s10_free_brand_data(zone_t *zone)
 }
 
 void
-s10_init_brand_data(zone_t *zone)
+s10_init_brand_data(zone_t *zone, kmutex_t *zsl)
 {
 	ASSERT(zone->zone_brand == &s10_brand);
 	ASSERT(zone->zone_brand_data == NULL);
diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c
index 33406eccc6..b1df79d006 100644
--- a/usr/src/uts/common/os/zone.c
+++ b/usr/src/uts/common/os/zone.c
@@ -2899,9 +2899,14 @@ zone_set_brand(zone_t *zone, const char *brand)
 		return (EINVAL);
 	}
 
-	/* set up the brand specific data */
+	/*
+	 * Set up the brand specific data.
+	 * Note that it's possible that the hook has to drop the
+	 * zone_status_lock and reaquire it before returning so we can't
+	 * assume the lock has been held the entire time.
+	 */
 	zone->zone_brand = bp;
-	ZBROP(zone)->b_init_brand_data(zone);
+	ZBROP(zone)->b_init_brand_data(zone, &zone_status_lock);
 
 	mutex_exit(&zone_status_lock);
 	return (0);
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index f5a46182b9..f8d7b9af39 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2015, Joyent, Inc.
+ * Copyright 2016, Joyent, Inc.
  */
 
 #ifndef _SYS_BRAND_H
@@ -150,7 +150,7 @@ struct execa;
  * b_pagefault - Trap pagefault events
  */
 struct brand_ops {
-	void	(*b_init_brand_data)(zone_t *);
+	void	(*b_init_brand_data)(zone_t *, kmutex_t *);
 	void	(*b_free_brand_data)(zone_t *);
 	int	(*b_brandsys)(int, int64_t *, uintptr_t, uintptr_t, uintptr_t,
 		uintptr_t, uintptr_t);