14434 overlays should persist across reboots

Reviewed by: Robert Mustacchi <rm+illumos@fingolfin.org>
Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Gordon Ross <gordon.w.ross@gmail.com>

12 files changed, 689 insertions, 139 deletions

diff --git a/usr/src/cmd/dladm/dladm.c b/usr/src/cmd/dladm/dladm.c
index 456811663e..e905111aea 100644
--- a/usr/src/cmd/dladm/dladm.c
+++ b/usr/src/cmd/dladm/dladm.c
@@ -233,7 +233,7 @@ static cmdfunc_t do_add_bridge, do_remove_bridge, do_show_bridge;
 static cmdfunc_t do_create_iptun, do_modify_iptun, do_delete_iptun;
 static cmdfunc_t do_show_iptun, do_up_iptun, do_down_iptun;
 static cmdfunc_t do_create_overlay, do_delete_overlay, do_modify_overlay;
-static cmdfunc_t do_show_overlay;
+static cmdfunc_t do_show_overlay, do_up_overlay;
 
 static void	do_up_vnic_common(int, char **, const char *, boolean_t);
 
@@ -424,13 +424,14 @@ static cmd_t	cmds[] = {
 	    "    create-overlay   [-t] -e <encap> -s <search> -v <vnetid>\n"
 	    "\t\t     [ -p <prop>=<value>[,...]] <overlay>"	},
 	{ "delete-overlay",	do_delete_overlay,
-	    "    delete-overlay   <overlay>"			},
+	    "    delete-overlay   [-t] <overlay>"			},
 	{ "modify-overlay",	do_modify_overlay,
 	    "    modify-overlay   -d mac | -f | -s mac=ip:port "
 	    "<overlay>"						},
 	{ "show-overlay",	do_show_overlay,
 	    "    show-overlay     [-f | -t] [[-p] -o <field>,...] "
 	    "[<overlay>]\n"						},
+	{ "up-overlay",		do_up_overlay,		NULL		},
 	{ "show-usage",		do_show_usage,
 	    "    show-usage       [-a] [-d | -F <format>] "
 	    "[-s <DD/MM/YYYY,HH:MM:SS>]\n"
@@ -9886,13 +9887,6 @@ do_create_overlay(int argc, char *argv[], const char *use)
 		}
 	}
 
-	/*
-	 * Overlays do not currently support persistence.
-	 * This will be addressed by https://www.illumos.org/issues/14434
-	 */
-	if ((flags & DLADM_OPT_PERSIST) != 0)
-		die("overlays do not (yet) support persistence, use -t");
-
 	if (havevid == B_FALSE)
 		die("missing required virtual network id");
 
@@ -9935,18 +9929,33 @@ do_delete_overlay(int argc, char *argv[], const char *use)
 {
 	datalink_id_t	linkid = DATALINK_ALL_LINKID;
 	dladm_status_t	status;
+	uint32_t flags = DLADM_OPT_ACTIVE | DLADM_OPT_PERSIST;
+	int option;
 
-	if (argc != 2) {
-		usage();
+	opterr = 0;
+	while ((option = getopt_long(argc, argv, ":t", lopts,
+	    NULL)) != -1) {
+		switch (option) {
+		case 't':
+			flags &= ~DLADM_OPT_PERSIST;
+			break;
+		default:
+			die_opterr(optopt, option, use);
+		}
 	}
 
-	status = dladm_name2info(handle, argv[1], &linkid, NULL, NULL, NULL);
+	/* get overlay name (required last argument) */
+	if (optind != (argc - 1))
+		usage();
+
+	status = dladm_name2info(handle, argv[optind], &linkid,
+	    NULL, NULL, NULL);
 	if (status != DLADM_STATUS_OK)
-		die_dlerr(status, "failed to delete %s", argv[1]);
+		die_dlerr(status, "failed to delete %s", argv[optind]);
 
-	status = dladm_overlay_delete(handle, linkid);
+	status = dladm_overlay_delete(handle, linkid, flags);
 	if (status != DLADM_STATUS_OK)
-		die_dlerr(status, "failed to delete %s", argv[1]);
+		die_dlerr(status, "failed to delete %s", argv[optind]);
 }
 
 typedef struct showoverlay_state {
@@ -10544,3 +10553,35 @@ do_modify_overlay(int argc, char *argv[], const char *use)
 	}
 
 }
+
+static void
+do_up_overlay(int argc, char *argv[], const char *use)
+{
+	datalink_id_t	linkid = DATALINK_ALL_LINKID;
+	dladm_status_t	status;
+
+	/*
+	 * get the id or the name of the overlay (optional last argument)
+	 */
+	if (argc == 2) {
+		status = dladm_name2info(handle, argv[1], &linkid, NULL, NULL,
+		    NULL);
+		if (status != DLADM_STATUS_OK)
+			goto done;
+	} else if (argc > 2) {
+		usage();
+	}
+
+	status = dladm_overlay_up(handle, linkid, &errlist);
+
+done:
+	if (status != DLADM_STATUS_OK) {
+		if (argc == 2) {
+			die_dlerrlist(status, &errlist,
+			    "could not bring up overlay '%s'", argv[1]);
+		} else {
+			die_dlerrlist(status, &errlist,
+			    "could not bring overlays up");
+		}
+	}
+}
diff --git a/usr/src/cmd/svc/milestone/net-nwam b/usr/src/cmd/svc/milestone/net-nwam
index fb012d7bee..d6b913bda1 100644
--- a/usr/src/cmd/svc/milestone/net-nwam
+++ b/usr/src/cmd/svc/milestone/net-nwam
@@ -22,6 +22,7 @@
 #
 # Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright 2012 Milan Jurik. All rights reserved.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 . /lib/svc/share/smf_include.sh
@@ -165,7 +166,7 @@ create_legacy_loc () {
 	# Copy /etc/nsswitch.conf file
 	copy_to_legacy_loc /etc/nsswitch.conf
 	NAMESERVICES_CONFIG_FILE="$VOL_LEGACY_PATH/nsswitch.conf"
-	
+
 	# Gather DNS info from resolv.conf if present.
 	if [ -f /etc/resolv.conf ]; then
 		NAMESERVICES="dns,"
@@ -215,12 +216,12 @@ create_legacy_loc () {
 
 	# Now, write nwamcfg commands for nameservices
 	write_loc_prop "nameservices" $NAMESERVICES $CREATE_LOC_LEGACY_FILE
- 	write_loc_prop "nameservices-config-file" $NAMESERVICES_CONFIG_FILE \
- 	    $CREATE_LOC_LEGACY_FILE
+	write_loc_prop "nameservices-config-file" $NAMESERVICES_CONFIG_FILE \
+	    $CREATE_LOC_LEGACY_FILE
 	write_loc_prop "dns-nameservice-configsrc" $DNS_NAMESERVICE_CONFIGSRC \
 	    $CREATE_LOC_LEGACY_FILE
 	write_loc_prop "dns-nameservice-domain" $DNS_NAMESERVICE_DOMAIN \
- 	    $CREATE_LOC_LEGACY_FILE
+	    $CREATE_LOC_LEGACY_FILE
 	write_loc_prop "dns-nameservice-servers" $DNS_NAMESERVICE_SERVERS \
 	    $CREATE_LOC_LEGACY_FILE
 	write_loc_prop "dns-nameservice-search" $DNS_NAMESERVICE_SEARCH \
@@ -238,7 +239,7 @@ create_legacy_loc () {
 	# Retrieve NFSv4 domain from SMF.
 	if service_is_enabled $NFS_MAPID_FMRI; then
 		NFS_DOMAIN=`get_smf_prop NFS_MAPID_FMRI \
-		    nfs-props/nfsmapid_domain`    
+		    nfs-props/nfsmapid_domain`
 		write_loc_prop "nfsv4-domain" \
 		    $NFS_DOMAIN $CREATE_LOC_LEGACY_FILE
 	fi
@@ -313,7 +314,7 @@ create_legacy_loc () {
 	if [ -n "$IPNAT_CONFIG_FILE" -a -f "$IPNAT_CONFIG_FILE" ]; then
 		write_loc_prop "ipnat-config-file" $IPNAT_CONFIG_FILE \
 		    $CREATE_LOC_LEGACY_FILE
-	fi   
+	fi
 	if [ -n "$IPPOOL_CONFIG_FILE" -a -f "$IPPOOL_CONFIG_FILE" ]; then
 		write_loc_prop "ippool-config-file" $IPPOOL_CONFIG_FILE \
 		    $CREATE_LOC_LEGACY_FILE
@@ -504,7 +505,7 @@ case "$1" in
 	# The real daemon is not started in a shared stack zone. But we need to
 	# create a dummy background process to preserve contract lifetime.
 	smf_configure_ip
-	if [ $? = "1" ] ; then 
+	if [ $? = "1" ] ; then
 		$RM -f $VOL_NWAM_PATH/nwam_blocked
 		$MKFIFO $VOL_NWAM_PATH/nwam_blocked
 		($CAT <$VOL_NWAM_PATH/nwam_blocked >/dev/null) &
@@ -565,6 +566,7 @@ case "$1" in
 		/sbin/dladm up-part
 		/sbin/dladm up-aggr
 		/sbin/flowadm init-flow
+		/sbin/dladm up-overlay
 	fi
 
 	#
@@ -588,7 +590,7 @@ case "$1" in
 'stop')
 	# We need to make the dummy process we created above stop.
 	smf_configure_ip
-	if [ $? = "1" ] ; then 
+	if [ $? = "1" ] ; then
 		echo "stop" > $VOL_NWAM_PATH/nwam_blocked
 		exit $SMF_EXIT_OK
 	fi
@@ -604,7 +606,7 @@ case "$1" in
 		exit $SMF_EXIT_OK
 	fi
 	revert_to_legacy_loc
-	# remove the location property group 
+	# remove the location property group
 	$SVCCFG -s $NET_LOC_FMRI delpg location
 	;;
 
@@ -614,7 +616,7 @@ case "$1" in
 	# network/physical:nwam will be disabled.
 	# There are various other parts of the system (nscd, nfs) that
 	# depend on continuing to have a working network.  For this
-	# reason we don't change the network configuration immediately.	
+	# reason we don't change the network configuration immediately.
 	#
 	# Disable network/physical temporarily and make sure that will
 	# be enabled on reboot.
@@ -622,7 +624,7 @@ case "$1" in
 	$SVCCFG -s $NET_PHYS_FMRI setprop general/enabled=true
 
 	# If nwam is online then make sure that it's temporarily enabled.
-	nwam_online=`$SVCPROP -t -p restarter/state $NET_NWAM_FMRI`	
+	nwam_online=`$SVCPROP -t -p restarter/state $NET_NWAM_FMRI`
 	if [ $? -eq 0 ]; then
 		set -- $nwam_online
 		[ $3 = "online" ] && $SVCADM enable -st $NET_NWAM_FMRI
diff --git a/usr/src/cmd/svc/milestone/net-physical b/usr/src/cmd/svc/milestone/net-physical
index ef72bee9e6..15929cb28a 100644
--- a/usr/src/cmd/svc/milestone/net-physical
+++ b/usr/src/cmd/svc/milestone/net-physical
@@ -27,6 +27,7 @@
 # Copyright 2012 Milan Jurik. All rights reserved.
 # Copyright 2021 Tintri by DDN, Inc. All rights reserved.
 #
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 
 . /lib/svc/share/smf_include.sh
 . /lib/svc/share/net_include.sh
@@ -39,7 +40,6 @@
 #
 smf_configure_ip || exit $SMF_EXIT_OK
 
-
 # Make sure that the libraries essential to this stage of booting can be found.
 LD_LIBRARY_PATH=/lib; export LD_LIBRARY_PATH
 
@@ -98,6 +98,15 @@ if smf_is_globalzone; then
 	# Create flows via flowadm.
 	#
 	/sbin/flowadm init-flow
+	#
+	# Bring up overlays
+	# Note that there may be VNICs configured over these overlays but
+	# these cannot be brought up until the network interface on which
+	# varpd will listen is enabled, and that doesn't happen until
+	# near the end of this script. Therefore VNIC initialisation is
+	# repeated below if overlays are present.
+	#
+	/sbin/dladm up-overlay
 fi
 
 #
@@ -148,6 +157,45 @@ is_iptun ()
 	return 1
 }
 
+bringup_ipif()
+{
+	for showif_output in `\
+	    /sbin/ipadm show-if -p -o ifname,state,current`; do
+		intf=`echo $showif_output | /usr/bin/cut -f1 -d:`
+		state=`echo $showif_output | /usr/bin/cut -f2 -d:`
+		current=`echo $showif_output | /usr/bin/cut -f3 -d:`
+		if  [[ "$state" != "disabled" && $current != *Z* ]]; then
+			#
+			# skip if not a persistent interface, or if it should
+			# get IP configuration from the global zone ('Z' flag
+			# is set)
+			#
+			continue;
+		elif is_iptun $intf; then
+			# skip IP tunnel interfaces plumbed by net-iptun
+			continue;
+		elif [ -f /etc/hostname.$intf ] || \
+		    [ -f /etc/hostname6.$intf ]; then
+			if [[ $current != *Z* ]]; then
+				echo "found /etc/hostname.$intf "\
+				    "or /etc/hostname6.$intf, "\
+				    "ignoring ipadm configuration" > /dev/msglog
+				continue;
+			else
+				echo "Ignoring /etc/hostname*.$intf" \
+				    > /dev/msglog
+				{
+					/sbin/ifconfig $intf unplumb
+					/sbin/ifconfig $intf inet6 unplumb
+				} > /dev/null 2>&1
+			fi
+		fi
+
+		# Enable the interface managed by ipadm
+		/sbin/ipadm enable-if -t $intf
+	done
+}
+
 #
 # All the IPv4 and IPv6 interfaces are plumbed before doing any
 # interface configuration.  This prevents errors from plumb failures
@@ -313,7 +361,7 @@ fi
 #
 if /usr/bin/grep -q _family /etc/ipadm/ipadm.conf; then
 	oldifs=$(/usr/bin/sed -En \
-	    's/^_ifname=([a-z0-9]+);_family=[0-9]+;$/\1/p' \
+	    's/^_ifname=([a-z0-9_]+);_family=[0-9]+;$/\1/p' \
 	    /etc/ipadm/ipadm.conf | /usr/bin/sort -u)
 	/usr/bin/sed -i '/_family/d' /etc/ipadm/ipadm.conf
 	for oldif in $oldifs; do
@@ -329,35 +377,7 @@ fi
 # we are in a non-global zone and Layer-3 protection of IP addresses is
 # enforced on the interface by the global zone.
 #
-for showif_output in `/sbin/ipadm show-if -p -o ifname,state,current`; do
-	intf=`echo $showif_output | /usr/bin/cut -f1 -d:`
-	state=`echo $showif_output | /usr/bin/cut -f2 -d:`
-	current=`echo $showif_output | /usr/bin/cut -f3 -d:`
-	if  [[ "$state" != "disabled" && $current != *Z* ]]; then
-		#
-		# skip if not a persistent interface, or if it should get IP
-		# configuration from the global zone ('Z' flag is set)
-		#
-		continue;
-	elif is_iptun $intf; then
-		# skip IP tunnel interfaces plumbed by net-iptun
-		continue;
-	elif [ -f /etc/hostname.$intf ] || [ -f /etc/hostname6.$intf ]; then
-		if [[ $current != *Z* ]]; then
-			echo "found /etc/hostname.$intf "\
-			    "or /etc/hostname6.$intf, "\
-			    "ignoring ipadm configuration" > /dev/msglog
-			continue;
-		else
-			echo "Ignoring /etc/hostname*.$intf" > /dev/msglog
-			/sbin/ifconfig $intf unplumb > /dev/null 2>&1
-			/sbin/ifconfig $intf inet6 unplumb > /dev/null 2>&1
-		fi
-	fi
-
-	# Enable the interface managed by ipadm
-	/sbin/ipadm enable-if -t $intf
-done
+bringup_ipif
 
 #
 # Process the /etc/hostname[6].* files for IPMP interfaces.  Processing these
@@ -440,6 +460,19 @@ if [ "$interface_names" != '/etc/dhcp.*[0-9]' ]; then
 	[ -n "$i4d_fail" ] && warn_failed_ifs "configure IPv4 DHCP" "$i4d_fail"
 fi
 
+# There is a chicken-and-egg problem with bringing up overlay VNICs at boot
+# time. When the first VNIC is added to an overlay, it creates a kernel socket
+# to listen for incoming encapsulated frames. Therefore, VNICs cannot be added
+# until after IP interfaces have been brought up. Overlay VNICs may themselves
+# have IP interfaces over them and so it is necessary to attempt to bring up
+# any remaining IP interfaces once the overlay VNICs are in place.
+if smf_is_globalzone && dladm show-link -p -o class | egrep -s 'overlay'; then
+	echo "Bringing up any remaining VNICs on overlays"
+	/sbin/dladm up-vnic
+	echo "Bringing up any remaining IP interfaces on overlay VNICs"
+	bringup_ipif
+fi
+
 # In order to avoid bringing up the interfaces that have
 # intentionally been left down, perform RARP only if the system
 # has no configured hostname in /etc/nodename
diff --git a/usr/src/cmd/varpd/varpd.xml b/usr/src/cmd/varpd/varpd.xml
index df7015a3d6..37d5754f83 100644
--- a/usr/src/cmd/varpd/varpd.xml
+++ b/usr/src/cmd/varpd/varpd.xml
@@ -11,21 +11,20 @@ source.  A copy of the CDDL is also available via the Internet at
 http://www.illumos.org/license/CDDL.
 
 Copyright 2018, Joyent, Inc.
+Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 -->
 
 <service_bundle type="manifest" name="illumos:varpd" >
 
 	<service name="network/varpd" type="service" version="1" >
 
-		<create_default_instance enabled="true" />
+		<create_default_instance enabled="false" />
 
-		<single_instance/>
-
-		<dependency name="varpd-network-physical"
+		<dependency name="varpd-datalink-management"
 			grouping="require_all"
 			restart_on="none"
 			type="service">
-			<service_fmri value="svc:/network/physical:default" />
+			<service_fmri value="svc:/network/datalink-management:default" />
 		</dependency>
 
 		<dependency name="varpd-device-local"
@@ -35,6 +34,19 @@ Copyright 2018, Joyent, Inc.
 			<service_fmri value="svc:/system/device/local:default" />
 		</dependency>
 
+		<dependency name="varpd-filesystem-minimal"
+			grouping="require_all"
+			restart_on="none"
+			type="service">
+			<service_fmri value="svc:/system/filesystem/minimal:default" />
+		</dependency>
+
+		<dependent name='varpd-network-physical-default'
+			grouping='optional_all'
+			restart_on='none'>
+			<service_fmri value='svc:/network/physical:default' />
+		</dependent>
+
 		<exec_method
 			type="method"
 			name="start"
diff --git a/usr/src/lib/libdladm/common/libdladm.c b/usr/src/lib/libdladm/common/libdladm.c
index b3e402fe2e..c2533469a7 100644
--- a/usr/src/lib/libdladm/common/libdladm.c
+++ b/usr/src/lib/libdladm/common/libdladm.c
@@ -1337,6 +1337,12 @@ dladm_errlist_reset(dladm_errlist_t *erl)
 	dladm_errlist_init(erl);
 }
 
+uint_t
+dladm_errlist_count(dladm_errlist_t *erl)
+{
+	return (erl->el_count);
+}
+
 dladm_status_t
 dladm_errlist_append(dladm_errlist_t *erl, const char *fmt, ...)
 {
diff --git a/usr/src/lib/libdladm/common/libdladm.h b/usr/src/lib/libdladm/common/libdladm.h
index a2e56047d2..66804308d1 100644
--- a/usr/src/lib/libdladm/common/libdladm.h
+++ b/usr/src/lib/libdladm/common/libdladm.h
@@ -309,6 +309,7 @@ extern int		dladm_range2strs(const mac_propval_range_t *, char **);
 extern dladm_status_t	dladm_list2range(void *, uint_t, mac_propval_type_t,
 			    mac_propval_range_t **);
 
+extern uint_t		dladm_errlist_count(dladm_errlist_t *);
 extern void		dladm_errlist_init(dladm_errlist_t *);
 extern void		dladm_errlist_reset(dladm_errlist_t *);
 
diff --git a/usr/src/lib/libdladm/common/libdladm_impl.h b/usr/src/lib/libdladm/common/libdladm_impl.h
index 9cd91d56c1..1502095de2 100644
--- a/usr/src/lib/libdladm/common/libdladm_impl.h
+++ b/usr/src/lib/libdladm/common/libdladm_impl.h
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef _LIBDLADM_IMPL_H
@@ -73,6 +73,14 @@ extern void		dladm_free_args(dladm_arg_list_t *);
 /*
  * Link attributes persisted by dlmgmtd.
  */
+
+/*
+ * Set for overlays only
+ */
+#define	FENCAP		"encap"		/* string */
+#define	FSEARCH		"search"	/* string */
+#define	FVNETID		"vnetid"	/* uint64_t */
+
 /*
  * Set for VLANs only
  */
@@ -120,7 +128,8 @@ extern void		dladm_free_args(dladm_arg_list_t *);
 /*
  * List of all the above attributes.
  */
-#define	DLADM_ATTR_NAMES	FVLANID, FLINKOVER, \
+#define	DLADM_ATTR_NAMES	FENCAP, FSEARCH, FVNETID, \
+				FVLANID, FLINKOVER, \
 				FKEY, FNPORTS, FPORTS, FPOLICY, \
 				FFIXMACADDR, FFORCE, FLACPMODE, FLACPTIMER, \
 				FMADDRTYPE, FMADDRLEN, FMADDRSLOT, \
diff --git a/usr/src/lib/libdladm/common/libdloverlay.c b/usr/src/lib/libdladm/common/libdloverlay.c
index ecb604fde9..3f92957bcf 100644
--- a/usr/src/lib/libdladm/common/libdloverlay.c
+++ b/usr/src/lib/libdladm/common/libdloverlay.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright (c) 2015 Joyent, Inc.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <libdladm_impl.h>
@@ -25,12 +26,84 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <limits.h>
+#include <libscf.h>
 #include <libvarpd_client.h>
 
 #define	VARPD_PROPERTY_NAME	"varpd/id"
+#define	VARPD_SERVICE		"network/varpd:default"
 
 static const char *dladm_overlay_doorpath = "/var/run/varpd/varpd.door";
 
+static boolean_t
+varpd_svc_isonline(void)
+{
+	boolean_t isonline = B_FALSE;
+	char *s;
+
+	if ((s = smf_get_state(VARPD_SERVICE)) != NULL) {
+		if (strcmp(s, SCF_STATE_STRING_ONLINE) == 0)
+			isonline = B_TRUE;
+		free(s);
+	}
+
+	return (isonline);
+}
+
+#define	MAX_WAIT_TIME	15
+
+static dladm_status_t
+varpd_enable_service(void)
+{
+	uint_t i;
+
+	if (varpd_svc_isonline())
+		return (DLADM_STATUS_OK);
+
+	if (smf_enable_instance(VARPD_SERVICE, 0) == -1) {
+		if (scf_error() == SCF_ERROR_PERMISSION_DENIED)
+			return (DLADM_STATUS_DENIED);
+		return (DLADM_STATUS_NOTFOUND);
+	}
+
+	/*
+	 * Wait up to MAX_WAIT_TIME seconds for the service
+	 */
+	for (i = 0; i < MAX_WAIT_TIME; i++) {
+		if (varpd_svc_isonline())
+			return (DLADM_STATUS_OK);
+		(void) sleep(1);
+	}
+	return (DLADM_STATUS_FAILED);
+}
+
+static int
+dladm_overlay_count_cb(dladm_handle_t handle, datalink_id_t linkid, void *arg)
+{
+	(*(uint32_t *)arg)++;
+	return (DLADM_WALK_CONTINUE);
+}
+
+/*
+ * Disable the varpd service if there are no overlays left.
+ */
+static void
+varpd_disable_service_when_no_overlays(dladm_handle_t handle)
+{
+	uint32_t cnt = 0;
+
+	/*
+	 * Get the number of the existing overlays. If there are no overlays
+	 * left, disable the service.
+	 */
+
+	(void) dladm_walk_datalink_id(dladm_overlay_count_cb, handle,
+	    &cnt, DATALINK_CLASS_OVERLAY, DATALINK_ANY_MEDIATYPE,
+	    DLADM_OPT_ACTIVE);
+
+	if (cnt == 0)
+		(void) smf_disable_instance(VARPD_SERVICE, 0);
+}
+
 typedef struct dladm_overlay_propinfo {
 	boolean_t dop_isvarpd;
 	union {
@@ -133,10 +206,35 @@ dladm_overlay_parse_prop(overlay_prop_type_t type, void *buf, uint32_t *sizep,
 	return (DLADM_STATUS_OK);
 }
 
-/* ARGSUSED */
 static dladm_status_t
-dladm_overlay_varpd_setprop(dladm_handle_t handle, varpd_client_handle_t *chdl,
-    uint64_t inst, const char *name, char *const *valp, uint_t cnt)
+i_dladm_overlay_setprop_db(dladm_handle_t handle, datalink_id_t linkid,
+    const char *name, char *const *valp, uint_t cnt)
+{
+	dladm_conf_t	conf;
+	dladm_status_t	status;
+
+	if (linkid == DATALINK_INVALID_LINKID ||
+	    name == NULL || valp == NULL || cnt != 1) {
+		return (DLADM_STATUS_BADARG);
+	}
+
+	status = dladm_open_conf(handle, linkid, &conf);
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	status = dladm_set_conf_field(handle, conf, name, DLADM_TYPE_STR,
+	    valp[0]);
+	if (status == DLADM_STATUS_OK)
+		status = dladm_write_conf(handle, conf);
+
+	dladm_destroy_conf(handle, conf);
+	return (status);
+}
+
+static dladm_status_t
+dladm_overlay_varpd_setprop(dladm_handle_t handle,
+    varpd_client_handle_t *chdl, uint64_t inst, datalink_id_t linkid,
+    const char *name, char *const *valp, uint_t cnt)
 {
 	int ret;
 	uint32_t size;
@@ -165,13 +263,19 @@ dladm_overlay_varpd_setprop(dladm_handle_t handle, varpd_client_handle_t *chdl,
 		return (status);
 	}
 
+	status = DLADM_STATUS_OK;
 	ret = libvarpd_c_prop_set(phdl, buf, size);
 	libvarpd_c_prop_handle_free(phdl);
+	if (ret != 0)
+		status = dladm_errno2status(ret);
 
-	return (dladm_errno2status(ret));
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	return (status);
 }
 
-dladm_status_t
+static dladm_status_t
 dladm_overlay_setprop(dladm_handle_t handle, datalink_id_t linkid,
     const char *name, char *const *valp, uint_t cnt)
 {
@@ -202,16 +306,19 @@ dladm_overlay_setprop(dladm_handle_t handle, datalink_id_t linkid,
 	prop.oip_linkid = linkid;
 	prop.oip_id = info.oipi_id;
 	prop.oip_name[0] = '\0';
-	if ((ret = dladm_overlay_parse_prop(info.oipi_type, prop.oip_value,
+	if ((status = dladm_overlay_parse_prop(info.oipi_type, prop.oip_value,
 	    &prop.oip_size, valp[0])) != DLADM_STATUS_OK)
-		return (ret);
+		return (status);
 
 	status = DLADM_STATUS_OK;
 	ret = ioctl(dladm_dld_fd(handle), OVERLAY_IOC_SETPROP, &prop);
 	if (ret != 0)
 		status = dladm_errno2status(errno);
 
-	return (ret);
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	return (status);
 }
 
 /*
@@ -256,18 +363,19 @@ dladm_overlay_activate_cb(dladm_handle_t handle, datalink_id_t linkid,
  * datalink id. If we fail to do that, then that name will become lost to time.
  */
 dladm_status_t
-dladm_overlay_delete(dladm_handle_t handle, datalink_id_t linkid)
+dladm_overlay_delete(dladm_handle_t handle, datalink_id_t linkid,
+    uint32_t flags)
 {
 	datalink_class_t class;
 	overlay_ioc_delete_t oid;
 	varpd_client_handle_t *chdl;
 	int ret;
-	uint32_t flags;
 	uint64_t varpdid;
 
-	if (dladm_datalink_id2info(handle, linkid, &flags, &class, NULL,
-	    NULL, 0) != DLADM_STATUS_OK)
+	if (dladm_datalink_id2info(handle, linkid, NULL, &class, NULL,
+	    NULL, 0) != DLADM_STATUS_OK) {
 		return (DLADM_STATUS_BADARG);
+	}
 
 	if (class != DATALINK_CLASS_OVERLAY)
 		return (DLADM_STATUS_BADARG);
@@ -293,7 +401,12 @@ dladm_overlay_delete(dladm_handle_t handle, datalink_id_t linkid)
 	ret = libvarpd_c_instance_destroy(chdl, varpdid);
 finish:
 	(void) libvarpd_c_destroy(chdl);
-	(void) dladm_destroy_datalink_id(handle, linkid, flags);
+	if ((flags & DLADM_OPT_PERSIST) != 0) {
+		(void) dladm_remove_conf(handle, linkid);
+		(void) dladm_destroy_datalink_id(handle, linkid, flags);
+	}
+
+	(void) varpd_disable_service_when_no_overlays(handle);
 
 	return (dladm_errno2status(ret));
 }
@@ -471,29 +584,50 @@ dladm_overlay_walk_prop(dladm_handle_t handle, datalink_id_t linkid,
 	return (ret);
 }
 
-dladm_status_t
-dladm_overlay_create(dladm_handle_t handle, const char *name,
-    const char *encap, const char *search, uint64_t vid,
-    dladm_arg_list_t *props, dladm_errlist_t *errs, uint32_t flags)
+static dladm_status_t
+dladm_overlay_persist_config(dladm_handle_t handle, dladm_overlay_attr_t *attr)
 {
-	int ret, i;
+	dladm_conf_t conf;
 	dladm_status_t status;
-	datalink_id_t linkid;
-	overlay_ioc_create_t oic;
-	overlay_ioc_activate_t oia;
-	size_t slen;
-	varpd_client_handle_t *vch;
-	uint64_t id;
 
-	status = dladm_create_datalink_id(handle, name, DATALINK_CLASS_OVERLAY,
-	    DL_ETHER, flags, &linkid);
-	if (status != DLADM_STATUS_OK)
+	if ((status = dladm_create_conf(handle, attr->oa_name, attr->oa_linkid,
+	    DATALINK_CLASS_OVERLAY, DL_ETHER, &conf)) != DLADM_STATUS_OK) {
 		return (status);
+	}
+
+	status = dladm_set_conf_field(handle, conf, FVNETID,
+	    DLADM_TYPE_UINT64, &attr->oa_vid);
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	status = dladm_set_conf_field(handle, conf, FENCAP,
+	    DLADM_TYPE_STR, attr->oa_encap);
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	status = dladm_set_conf_field(handle, conf, FSEARCH,
+	    DLADM_TYPE_STR, attr->oa_search);
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	status = dladm_write_conf(handle, conf);
+
+done:
+	dladm_destroy_conf(handle, conf);
+	return (status);
+}
+
+static dladm_status_t
+i_dladm_overlay_create_sys(dladm_handle_t handle, dladm_overlay_attr_t *attr)
+{
+	overlay_ioc_create_t oic;
+	dladm_status_t status;
+	int ret;
 
 	bzero(&oic, sizeof (oic));
-	oic.oic_linkid = linkid;
-	oic.oic_vnetid = vid;
-	(void) strlcpy(oic.oic_encap, encap, MAXLINKNAMELEN);
+	oic.oic_linkid = attr->oa_linkid;
+	oic.oic_vnetid = attr->oa_vid;
+	(void) strlcpy(oic.oic_encap, attr->oa_encap, MAXLINKNAMELEN);
 
 	status = DLADM_STATUS_OK;
 	ret = ioctl(dladm_dld_fd(handle), OVERLAY_IOC_CREATE, &oic);
@@ -505,12 +639,22 @@ dladm_overlay_create(dladm_handle_t handle, const char *name,
 		status = dladm_errno2status(errno);
 	}
 
-	if (status != DLADM_STATUS_OK) {
-		(void) dladm_destroy_datalink_id(handle, linkid, flags);
-		return (status);
-	}
+	return (status);
+}
+
+static dladm_status_t
+i_dladm_overlay_commit_sys(dladm_handle_t handle, dladm_overlay_attr_t *attr,
+    dladm_arg_list_t *props, dladm_errlist_t *errs)
+{
+	overlay_ioc_activate_t oia;
+	varpd_client_handle_t *vch;
+	dladm_status_t status;
+	size_t slen;
+	uint64_t id;
+	int ret;
+	uint_t i;
 
-	slen = strlen(search);
+	slen = strlen(attr->oa_search);
 	for (i = 0; props != NULL && i < props->al_count; i++) {
 		dladm_arg_info_t	*aip = &props->al_info[i];
 
@@ -519,33 +663,41 @@ dladm_overlay_create(dladm_handle_t handle, const char *name,
 		 * prefix '<search>/', then we don't set the property on the
 		 * overlay device and instead set it on the varpd instance.
 		 */
-		if (strncmp(aip->ai_name, search, slen) == 0 &&
+		if (strncmp(aip->ai_name, attr->oa_search, slen) == 0 &&
 		    aip->ai_name[slen] == '/')
 			continue;
-		status = dladm_overlay_setprop(handle, linkid, aip->ai_name,
-		    aip->ai_val, aip->ai_count);
+		status = dladm_overlay_setprop(handle, attr->oa_linkid,
+		    aip->ai_name, aip->ai_val, aip->ai_count);
 		if (status != DLADM_STATUS_OK) {
 			(void) dladm_errlist_append(errs,
-			    "failed to set property %s",
-			    aip->ai_name);
-			(void) dladm_overlay_delete(handle, linkid);
+			    "failed to set property %s", aip->ai_name);
 			return (status);
 		}
+
+		if (attr->oa_flags & DLADM_OPT_PERSIST) {
+			status = i_dladm_overlay_setprop_db(handle,
+			    attr->oa_linkid, aip->ai_name, aip->ai_val,
+			    aip->ai_count);
+			if (status != DLADM_STATUS_OK) {
+				(void) dladm_errlist_append(errs,
+				    "failed to persistently set property %s",
+				    aip->ai_name);
+				return (status);
+			}
+		}
 	}
 
 	if ((ret = libvarpd_c_create(&vch, dladm_overlay_doorpath)) != 0) {
 		(void) dladm_errlist_append(errs,
 		    "failed to create libvarpd handle: %s", strerror(ret));
-		(void) dladm_overlay_delete(handle, linkid);
 		return (dladm_errno2status(ret));
 	}
 
-	if ((ret = libvarpd_c_instance_create(vch, linkid, search,
-	    &id)) != 0) {
+	if ((ret = libvarpd_c_instance_create(vch, attr->oa_linkid,
+	    attr->oa_search, &id)) != 0) {
 		(void) dladm_errlist_append(errs,
 		    "failed to create varpd instance: %s", strerror(ret));
 		libvarpd_c_destroy(vch);
-		(void) dladm_overlay_delete(handle, linkid);
 		return (dladm_errno2status(ret));
 	}
 
@@ -555,39 +707,48 @@ dladm_overlay_create(dladm_handle_t handle, const char *name,
 		/*
 		 * Skip arguments we've processed already.
 		 */
-		if (strncmp(aip->ai_name, search, slen) != 0)
-			continue;
-
-		if (aip->ai_name[slen] != '/')
+		if (strncmp(aip->ai_name, attr->oa_search, slen) != 0 ||
+		    aip->ai_name[slen] != '/')
 			continue;
 
-		ret = dladm_overlay_varpd_setprop(handle, vch, id, aip->ai_name,
-		    aip->ai_val, aip->ai_count);
+		ret = dladm_overlay_varpd_setprop(handle, vch, id,
+		    attr->oa_linkid, aip->ai_name, aip->ai_val, aip->ai_count);
 		if (ret != 0) {
 			(void) dladm_errlist_append(errs,
-			    "failed to set varpd prop: %s\n",
-			    aip->ai_name);
+			    "failed to set varpd prop: %s\n", aip->ai_name);
 			(void) libvarpd_c_instance_destroy(vch, id);
 			libvarpd_c_destroy(vch);
-			(void) dladm_overlay_delete(handle, linkid);
 			return (dladm_errno2status(ret));
 		}
+
+		if (attr->oa_flags & DLADM_OPT_PERSIST) {
+			status = i_dladm_overlay_setprop_db(handle,
+			    attr->oa_linkid, aip->ai_name, aip->ai_val,
+			    aip->ai_count);
+			if (status != DLADM_STATUS_OK) {
+				(void) dladm_errlist_append(errs, "failed to "
+				    "persistently set varpd prop: %s\n",
+				    aip->ai_name);
+				(void) libvarpd_c_instance_destroy(vch, id);
+				libvarpd_c_destroy(vch);
+				return (status);
+			}
+		}
 	}
 
 	if ((ret = libvarpd_c_instance_activate(vch, id)) != 0) {
 		(void) dladm_errlist_append(errs,
 		    "failed to activate varpd instance: %s", strerror(ret));
-		(void) dladm_overlay_walk_varpd_prop(handle, linkid, id,
-		    dladm_overlay_activate_cb, errs);
+		(void) dladm_overlay_walk_varpd_prop(handle, attr->oa_linkid,
+		    id, dladm_overlay_activate_cb, errs);
 		(void) libvarpd_c_instance_destroy(vch, id);
 		libvarpd_c_destroy(vch);
-		(void) dladm_overlay_delete(handle, linkid);
 		return (dladm_errno2status(ret));
 
 	}
 
 	bzero(&oia, sizeof (oia));
-	oia.oia_linkid = linkid;
+	oia.oia_linkid = attr->oa_linkid;
 	status = DLADM_STATUS_OK;
 	ret = ioctl(dladm_dld_fd(handle), OVERLAY_IOC_ACTIVATE, &oia);
 	if (ret != 0) {
@@ -595,20 +756,78 @@ dladm_overlay_create(dladm_handle_t handle, const char *name,
 		(void) dladm_errlist_append(errs, "failed to activate "
 		    "device: %s", strerror(ret));
 		(void) libvarpd_c_instance_destroy(vch, id);
-		(void) dladm_overlay_walk_prop(handle, linkid,
+		(void) dladm_overlay_walk_prop(handle, attr->oa_linkid,
 		    dladm_overlay_activate_cb, errs, errs);
 		status = dladm_errno2status(ret);
-		(void) libvarpd_c_instance_destroy(vch, id);
 	}
 
 	libvarpd_c_destroy(vch);
-	if (status != DLADM_STATUS_OK)
-		(void) dladm_overlay_delete(handle, linkid);
 
 	return (status);
 }
 
+dladm_status_t
+dladm_overlay_create(dladm_handle_t handle, const char *name,
+    const char *encap, const char *search, uint64_t vid,
+    dladm_arg_list_t *props, dladm_errlist_t *errs, uint32_t flags)
+{
+	dladm_status_t status;
+	datalink_id_t linkid;
+	dladm_overlay_attr_t attr;
+	char errmsg[DLADM_STRSIZE];
+
+	if (strlcpy(attr.oa_name, name, sizeof (attr.oa_name)) >=
+	    sizeof (attr.oa_name)) {
+		return (DLADM_STATUS_BADARG);
+	}
+	if (strlcpy(attr.oa_encap, encap, sizeof (attr.oa_encap)) >=
+	    sizeof (attr.oa_encap)) {
+		return (DLADM_STATUS_BADARG);
+	}
+	if (strlcpy(attr.oa_search, search, sizeof (attr.oa_search)) >=
+	    sizeof (attr.oa_search)) {
+		return (DLADM_STATUS_BADARG);
+	}
+
+	status = varpd_enable_service();
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	status = dladm_create_datalink_id(handle, name, DATALINK_CLASS_OVERLAY,
+	    DL_ETHER, flags, &linkid);
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	attr.oa_linkid = linkid;
+	attr.oa_vid = vid;
+	attr.oa_flags = flags;
+
+	status = i_dladm_overlay_create_sys(handle, &attr);
+
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_destroy_datalink_id(handle, linkid, flags);
+		return (status);
+	}
+
+	if ((flags & DLADM_OPT_PERSIST) != 0) {
+		status = dladm_overlay_persist_config(handle, &attr);
+		if (status != DLADM_STATUS_OK) {
+			(void) dladm_errlist_append(errs, "failed to create "
+			    "persistent configuration for %s: %s",
+			    attr.oa_name, dladm_status2str(status, errmsg));
+		}
+	}
+
+	if (status == DLADM_STATUS_OK)
+		status = i_dladm_overlay_commit_sys(handle, &attr, props, errs);
+
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_overlay_delete(handle, linkid, flags);
+		(void) dladm_destroy_datalink_id(handle, linkid, flags);
+	}
 
+	return (status);
+}
 
 typedef struct overlay_walk_cb {
 	dladm_handle_t		owc_handle;
@@ -619,7 +838,6 @@ typedef struct overlay_walk_cb {
 	uint_t			owc_dest;
 } overlay_walk_cb_t;
 
-/* ARGSUSED */
 static int
 dladm_overlay_walk_cache_cb(varpd_client_handle_t *chdl, uint64_t varpdid,
     const struct ether_addr *key, const varpd_client_cache_entry_t *entry,
@@ -681,7 +899,6 @@ dladm_overlay_walk_cache(dladm_handle_t handle, datalink_id_t linkid,
 	return (dladm_errno2status(ret));
 }
 
-/* ARGSUSED */
 dladm_status_t
 dladm_overlay_cache_flush(dladm_handle_t handle, datalink_id_t linkid)
 {
@@ -703,7 +920,6 @@ dladm_overlay_cache_flush(dladm_handle_t handle, datalink_id_t linkid)
 	return (dladm_errno2status(ret));
 }
 
-/* ARGSUSED */
 dladm_status_t
 dladm_overlay_cache_delete(dladm_handle_t handle, datalink_id_t linkid,
     const struct ether_addr *key)
@@ -726,7 +942,6 @@ dladm_overlay_cache_delete(dladm_handle_t handle, datalink_id_t linkid,
 	return (dladm_errno2status(ret));
 }
 
-/* ARGSUSED */
 dladm_status_t
 dladm_overlay_cache_set(dladm_handle_t handle, datalink_id_t linkid,
     const struct ether_addr *key, char *val)
@@ -840,7 +1055,6 @@ send:
 	return (dladm_errno2status(ret));
 }
 
-/* ARGSUSED */
 dladm_status_t
 dladm_overlay_cache_get(dladm_handle_t handle, datalink_id_t linkid,
     const struct ether_addr *key, dladm_overlay_point_t *point)
@@ -904,3 +1118,212 @@ dladm_overlay_status(dladm_handle_t handle, datalink_id_t linkid,
 	func(handle, linkid, &dos, arg);
 	return (DLADM_STATUS_OK);
 }
+
+/*
+ * dladm_parse_args() usually creates a dladm_arg_list_t by tokenising a
+ * delimited string and storing pointers to pieces of that string in the
+ * dladm_arg_info_t structure. Those pointers do not need to be individually
+ * freed.
+ *
+ * This function deals with property lists which have instead been built from
+ * the persistent datalink configuration database, in order to bring up an
+ * overlay at boot time. In this case, the properties have been retrieved
+ * one-by-one, duplicated with strdup(), and added to the list. When the list
+ * is finished with, this function takes care of freeing the memory.
+ */
+static void
+i_dladm_overlay_props_free(dladm_handle_t handle, dladm_arg_list_t *props)
+{
+	uint_t i, j;
+
+	for (i = 0; props != NULL && i < props->al_count; i++) {
+		dladm_arg_info_t *aip = &props->al_info[i];
+
+		/* For ai_name, we need to cast away the 'const' qualifier. */
+		free((char *)aip->ai_name);
+		for (j = 0; j < aip->ai_count; j++)
+			free(aip->ai_val[j]);
+	}
+	free(props);
+}
+
+static dladm_status_t
+i_dladm_overlay_fetch_persistent_config(dladm_handle_t handle,
+    datalink_id_t linkid, dladm_overlay_attr_t *attrp,
+    dladm_arg_list_t **props)
+{
+	dladm_conf_t conf;
+	dladm_status_t status;
+	char attr[MAXLINKATTRLEN], last_attr[MAXLINKATTRLEN];
+	char attrval[OVERLAY_PROP_SIZEMAX];
+	size_t attrsz;
+	dladm_arg_list_t *list = NULL;
+
+	*props = NULL;
+
+	if ((status = dladm_getsnap_conf(handle, linkid, &conf)) !=
+	    DLADM_STATUS_OK) {
+		return (status);
+	}
+
+	attrp->oa_linkid = linkid;
+
+	status = dladm_get_conf_field(handle, conf, FVNETID, &attrp->oa_vid,
+	    sizeof (attrp->oa_vid));
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	status = dladm_get_conf_field(handle, conf, FENCAP,
+	    attrp->oa_encap, sizeof (attrp->oa_encap));
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	status = dladm_get_conf_field(handle, conf, FSEARCH,
+	    attrp->oa_search, sizeof (attrp->oa_search));
+	if (status != DLADM_STATUS_OK)
+		goto done;
+
+	list = calloc(1, sizeof (dladm_arg_list_t));
+
+	*last_attr = '\0';
+	while (dladm_getnext_conf_linkprop(handle, conf, last_attr,
+	    attr, attrval, sizeof (attrval), &attrsz) == DLADM_STATUS_OK) {
+		dladm_arg_info_t *aip;
+
+		(void) strlcpy(last_attr, attr, sizeof (last_attr));
+		if (strchr(attr, '/') == NULL)
+			continue;
+
+		aip = &list->al_info[list->al_count];
+		bzero(aip, sizeof (dladm_arg_info_t));
+		if ((aip->ai_name = strdup(attr)) == NULL) {
+			status = dladm_errno2status(errno);
+			break;
+		}
+		if ((aip->ai_val[0] = strdup(attrval)) == NULL) {
+			status = dladm_errno2status(errno);
+			break;
+		}
+		aip->ai_count = 1;
+		list->al_count++;
+		if (list->al_count >= DLADM_MAX_ARG_CNT) {
+			status = DLADM_STATUS_TOOMANYELEMENTS;
+			break;
+		}
+	}
+
+done:
+
+	dladm_destroy_conf(handle, conf);
+
+	if (status != DLADM_STATUS_OK) {
+		if (list != NULL)
+			i_dladm_overlay_props_free(handle, list);
+		return (status);
+	}
+
+	*props = list;
+	return (DLADM_STATUS_OK);
+}
+
+typedef struct dladm_overlay_up_arg_s {
+	dladm_errlist_t	*errlist;
+} dladm_overlay_up_arg_t;
+
+static int
+i_dladm_overlay_up(dladm_handle_t handle, datalink_id_t linkid, void *arg)
+{
+	dladm_overlay_up_arg_t *argp = arg;
+	dladm_errlist_t *errs = argp->errlist;
+	datalink_class_t class;
+	dladm_status_t status;
+	dladm_overlay_attr_t attr;
+	dladm_arg_list_t *props;
+	char errmsg[DLADM_STRSIZE];
+
+	bzero(&attr, sizeof (attr));
+
+	status = dladm_datalink_id2info(handle, linkid, NULL, &class,
+	    NULL, attr.oa_name, sizeof (attr.oa_name));
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs, "failed to get info for "
+		    "datalink id %u: %s",
+		    linkid, dladm_status2str(status, errmsg));
+		return (DLADM_STATUS_BADARG);
+	}
+
+	if (class != DATALINK_CLASS_OVERLAY) {
+		(void) dladm_errlist_append(errs, "%s is not an overlay",
+		    attr.oa_name);
+		return (DLADM_STATUS_BADARG);
+	}
+
+	status = varpd_enable_service();
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs, "failed to enable svc:/%s",
+		    VARPD_SERVICE);
+		return (DLADM_WALK_TERMINATE);
+	}
+
+	status = i_dladm_overlay_fetch_persistent_config(handle, linkid,
+	    &attr, &props);
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs, "failed to retrieve "
+		    "persistent configuration for %s: %s",
+		    attr.oa_name, dladm_status2str(status, errmsg));
+		return (DLADM_WALK_CONTINUE);
+	}
+
+	status = i_dladm_overlay_create_sys(handle, &attr);
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs,
+		    "failed to create overlay device %s: %s",
+		    attr.oa_name, dladm_status2str(status, errmsg));
+		goto out;
+	}
+
+	status = i_dladm_overlay_commit_sys(handle, &attr, props, errs);
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs,
+		    "failed to set properties for overlay device %s: %s",
+		    attr.oa_name, dladm_status2str(status, errmsg));
+		dladm_overlay_delete(handle, linkid, 0);
+		goto out;
+	}
+
+	status = dladm_up_datalink_id(handle, linkid);
+	if (status != DLADM_STATUS_OK) {
+		(void) dladm_errlist_append(errs,
+		    "failed to bring datalink up for overlay device %s: %s",
+		    attr.oa_name, dladm_status2str(status, errmsg));
+		dladm_overlay_delete(handle, linkid, 0);
+		goto out;
+	}
+
+out:
+	i_dladm_overlay_props_free(handle, props);
+
+	return (DLADM_WALK_CONTINUE);
+}
+
+dladm_status_t
+dladm_overlay_up(dladm_handle_t handle, datalink_id_t linkid,
+    dladm_errlist_t *errs)
+{
+	dladm_overlay_up_arg_t overlay_arg = {
+		.errlist	= errs
+	};
+
+	if (linkid == DATALINK_ALL_LINKID) {
+		(void) dladm_walk_datalink_id(i_dladm_overlay_up, handle,
+		    &overlay_arg, DATALINK_CLASS_OVERLAY,
+		    DATALINK_ANY_MEDIATYPE, DLADM_OPT_PERSIST);
+	} else {
+		(void) i_dladm_overlay_up(handle, linkid, &overlay_arg);
+	}
+
+	if (dladm_errlist_count(errs) == 0)
+		return (DLADM_STATUS_OK);
+
+	return (DLADM_STATUS_FAILED);
+}
diff --git a/usr/src/lib/libdladm/common/libdloverlay.h b/usr/src/lib/libdladm/common/libdloverlay.h
index 39b01ccae3..d8d0d59952 100644
--- a/usr/src/lib/libdladm/common/libdloverlay.h
+++ b/usr/src/lib/libdladm/common/libdloverlay.h
@@ -11,6 +11,7 @@
 
 /*
  * Copyright (c) 2015 Joyent, Inc.
+ * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #ifndef _LIBDLOVERLAY_H
@@ -28,6 +29,15 @@
 extern "C" {
 #endif
 
+typedef struct dladm_overlay_attr {
+	datalink_id_t		oa_linkid;
+	char			oa_name[MAXLINKNAMELEN];
+	char			oa_encap[OVERLAY_PROP_SIZEMAX];
+	char			oa_search[OVERLAY_PROP_SIZEMAX];
+	uint64_t		oa_vid;
+	uint32_t		oa_flags;
+} dladm_overlay_attr_t;
+
 #define	DLADM_OVERLAY_F_DROP	0x0001
 #define	DLADM_OVERLAY_F_DEFAULT	0xf000
 
@@ -47,7 +57,10 @@ typedef struct dladm_overlay_status {
 extern dladm_status_t dladm_overlay_create(dladm_handle_t, const char *,
     const char *, const char *, uint64_t, dladm_arg_list_t *, dladm_errlist_t *,
     uint32_t);
-extern dladm_status_t dladm_overlay_delete(dladm_handle_t, datalink_id_t);
+extern dladm_status_t dladm_overlay_delete(dladm_handle_t, datalink_id_t,
+    uint32_t);
+extern dladm_status_t dladm_overlay_up(dladm_handle_t, datalink_id_t,
+    dladm_errlist_t *);
 
 typedef void (*dladm_overlay_status_f)(dladm_handle_t, datalink_id_t,
     dladm_overlay_status_t *, void *);
diff --git a/usr/src/lib/libdladm/common/mapfile-vers b/usr/src/lib/libdladm/common/mapfile-vers
index eba6118ace..9cf199b724 100644
--- a/usr/src/lib/libdladm/common/mapfile-vers
+++ b/usr/src/lib/libdladm/common/mapfile-vers
@@ -21,6 +21,7 @@
 #
 # Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright 2015 Joyent, Inc.
+# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -281,6 +282,7 @@ SYMBOL_VERSION SUNWprivate_1.1 {
 	dladm_overlay_prop_info;
 	dladm_overlay_get_prop;
 	dladm_overlay_walk_prop;
+	dladm_overlay_up;
 
 	dladm_overlay_cache_set;
 	dladm_overlay_cache_get;
diff --git a/usr/src/man/man8/dladm.8 b/usr/src/man/man8/dladm.8
index f7ac9ab8d6..b18a9d7f50 100644
--- a/usr/src/man/man8/dladm.8
+++ b/usr/src/man/man8/dladm.8
@@ -43,9 +43,9 @@
 .\" Copyright (c) 2008, Sun Microsystems, Inc. All Rights Reserved
 .\" Copyright 2016 Joyent, Inc.
 .\" Copyright 2020 RackTop Systems, Inc.
-.\" Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
+.\" Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
 .\"
-.TH DLADM 8 "October 20, 2021"
+.TH DLADM 8 "February 01, 2022"
 .SH NAME
 dladm \- administer data links
 .SH SYNOPSIS
@@ -179,7 +179,7 @@ dladm \- administer data links
 .LP
 .nf
 \fBdladm create-overlay\fR [\fB-t\fR] \fB-e\fR \fIencap\fR \fB-s\fR \fIsearch\fR \fB-v\fR \fIvnetid\fR [\fB-p\fR \fIprop\fR=\fIvalue\fR[,...]] \fIoverlay\fR
-\fBdladm delete-overlay\fR \fIoverlay\fR
+\fBdladm delete-overlay\fR [\fB-t\fR] \fIoverlay\fR
 \fBdladm modify-overlay\fR \fB-d\fR \fImac\fR | \fB-f\fR | \fB-s\fR \fImac=ip:port\fR \fIoverlay\fR
 \fBdladm show-overlay\fR [ \fB-f\fR | \fB-t\fR ] [[\fB-p\fR] \fB-o\fR \fIfield\fR[,...]] [\fIoverlay\fR]
 .fi
@@ -4392,8 +4392,7 @@ The tunnel destination address.
 .sp
 .ne 2
 .na
-\fBdladm create-overlay\fR \fB-e\fR \fIencap\fR \fB-s\fR \fIsearch\fR
-\fB-v\fR \fIvnetid\fR [\fB-p\fR \fIprop\fR=\fIvalue\fR[,...]] \fIoverlay\fR
+\fBdladm create-overlay\fR [\fB-t\fR] \fB-e\fR \fIencap\fR \fB-s\fR \fIsearch\fR \fB-v\fR \fIvnetid\fR [\fB-p\fR \fIprop\fR=\fIvalue\fR[,...]] \fIoverlay\fR
 .ad
 .sp .6
 .RS 4n
@@ -4477,12 +4476,21 @@ determined by the encapsulation plugin specified by \fB-e\fR.
 .sp
 .ne 2
 .na
-\fBdladm delete-overlay\fR \fIoverlay\fR
+\fBdladm delete-overlay\fR [\fB-t\fR] \fIoverlay\fR
 .ad
 .sp .6
 .RS 4n
 Delete the specified overlay. This will fail if there are VNICs on top of the
 device.
+.sp
+.ne 2
+.na
+\fB-t\fR, \fB--temporary\fR
+.ad
+.sp .6
+.RS 4n
+Specifies that the deletion is temporary. Temporary deletions last until the
+next reboot.
 .RE
 
 .sp
diff --git a/usr/src/uts/common/sys/overlay_common.h b/usr/src/uts/common/sys/overlay_common.h
index 5c4b651f2c..894644b50a 100644
--- a/usr/src/uts/common/sys/overlay_common.h
+++ b/usr/src/uts/common/sys/overlay_common.h
@@ -42,7 +42,7 @@ typedef enum overlay_prop_type {
 	OVERLAY_PROP_T_INT = 0x1,	/* signed int */
 	OVERLAY_PROP_T_UINT,		/* unsigned int */
 	OVERLAY_PROP_T_IP,		/* sinaddr6 */
-	OVERLAY_PROP_T_STRING		/* OVERLAY_PROPS_SIZEMAX */
+	OVERLAY_PROP_T_STRING		/* OVERLAY_PROP_SIZEMAX */
 } overlay_prop_type_t;
 
 typedef enum overlay_prop_prot {