[illumos-gate merge]

commit a73be61a80f7331c35adfa540bcf8f1546ff1e33
    2554 ipadm needs IPMP configuration support
commit f81209f5137586c57e31f7d74b929149299d9b3c
    14247 SMF seed databases contain build paths
commit 671e12f2cf68dbd241dfc08296dc05fecc27127c
    14235 pcitool doesn't properly handle AMD Type 1 I/O space access
commit b15548ab7300ec8625be47442fd92b71ed91c2cd
    14245 sed regression tests 7.1 and 7.7 are locale sensitive
commit 0153d828c132fdb1a17c11b99386a3d1b87994cf
    13896 bhyve VM interfaces should be better fit
    13981 bhyve emulation should set dirty bits
commit d8f839f91e21bea2f5200f95df55608cbecdeeb9
    14223 Add change key zfs channel program
commit 705b6680745618ebbf67feb254ce9a62511084a5
    4450 pointless condfree in libfmnotify

Conflicts:
    usr/src/cmd/cmd-inet/lib/ipmgmtd/Makefile
    usr/src/cmd/cmd-inet/lib/ipmgmtd/ipmgmt_main.c
    usr/src/cmd/svc/milestone/net-physical
    usr/src/lib/libipadm/common/libipadm.c
    usr/src/uts/common/fs/zfs/zfs_ioctl.c

10 files changed, 1146 insertions, 329 deletions

diff --git a/usr/src/lib/libipadm/Makefile.com b/usr/src/lib/libipadm/Makefile.com
index 80c052fffd..aed875db48 100644
--- a/usr/src/lib/libipadm/Makefile.com
+++ b/usr/src/lib/libipadm/Makefile.com
@@ -22,6 +22,7 @@
 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
 # Copyright (c) 2018, Joyent, Inc.
+# Copyright 2021 Tintri by DDN, Inc. All rights reserved.
 #
 
 LIBRARY = libipadm.a
@@ -36,7 +37,7 @@ include ../../Makefile.rootfs
 
 LIBS =		$(DYNLIB)
 LDLIBS +=	-lc -linetutil -lsocket -ldlpi -lnvpair -ldhcpagent \
-		-ldladm -lsecdb -ldhcputil
+		-ldladm -lsecdb -ldhcputil -lipmp -lcmdutils
 
 SRCDIR =	../common
 
diff --git a/usr/src/lib/libipadm/common/ipadm_addr.c b/usr/src/lib/libipadm/common/ipadm_addr.c
index 4bca7ecb4f..6bdfc487d3 100644
--- a/usr/src/lib/libipadm/common/ipadm_addr.c
+++ b/usr/src/lib/libipadm/common/ipadm_addr.c
@@ -18,10 +18,12 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2016-2017, Chris Fraire <cfraire@me.com>.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 /*
@@ -282,8 +284,11 @@ i_ipadm_get_static_addr_db(ipadm_handle_t iph, ipadm_addrobj_t ipaddr)
 		    nvlist_exists(anvl, IPADM_NVP_IPV6ADDR))
 			break;
 	}
+	nvlist_free(onvl);
+
 	if (nvp == NULL)
-		goto fail;
+		return (IPADM_NOTFOUND);
+
 	for (nvp = nvlist_next_nvpair(anvl, NULL);
 	    nvp != NULL; nvp = nvlist_next_nvpair(anvl, nvp)) {
 		name = nvpair_name(nvp);
@@ -296,16 +301,13 @@ i_ipadm_get_static_addr_db(ipadm_handle_t iph, ipadm_addrobj_t ipaddr)
 		}
 	}
 	assert(af != AF_UNSPEC);
+
 	if (nvpair_value_nvlist(nvp, &nvladdr) != 0 ||
 	    nvlist_lookup_string(nvladdr, IPADM_NVP_IPADDRHNAME, &sname) != 0 ||
-	    ipadm_set_addr(ipaddr, sname, af) != IPADM_SUCCESS) {
-		goto fail;
-	}
-	nvlist_free(onvl);
+	    ipadm_set_addr(ipaddr, sname, af) != IPADM_SUCCESS)
+		return (IPADM_NOTFOUND);
+
 	return (IPADM_SUCCESS);
-fail:
-	nvlist_free(onvl);
-	return (IPADM_NOTFOUND);
 }
 
 /*
@@ -383,7 +385,7 @@ ipadm_delete_aobjname(ipadm_handle_t iph, const char *ifname, sa_family_t af,
  * Gets all the addresses from active configuration and populates the
  * address information in `addrinfo'.
  */
-static ipadm_status_t
+ipadm_status_t
 i_ipadm_active_addr_info(ipadm_handle_t iph, const char *ifname,
     ipadm_addr_info_t **addrinfo, uint32_t ipadm_flags, int64_t lifc_flags)
 {
@@ -1713,7 +1715,6 @@ ipadm_set_addrprop(ipadm_handle_t iph, const char *pname,
 	if ((pflags & IPADM_OPT_PERSIST) &&
 	    !(ipaddr.ipadm_flags & IPMGMT_PERSIST))
 		return (IPADM_TEMPORARY_OBJ);
-
 	/*
 	 * Currently, setting an address property on an address object of type
 	 * IPADM_ADDR_IPV6_ADDRCONF is not supported. Supporting it involves
@@ -2409,11 +2410,9 @@ ipadm_create_addrobj(ipadm_addr_type_t type, const char *aobjname,
 	/* Check if the interface name is valid. */
 	if (!ifparse_ifspec(ifname, &ifsp))
 		return (IPADM_INVALID_ARG);
-
 	/* Check if the given addrobj name is valid. */
 	if (aname != NULL && !i_ipadm_is_user_aobjname_valid(aname))
 		return (IPADM_INVALID_ARG);
-
 	if ((newaddr = calloc(1, sizeof (struct ipadm_addrobj_s))) == NULL)
 		return (IPADM_NO_MEMORY);
 
@@ -2595,10 +2594,6 @@ i_ipadm_get_db_addr(ipadm_handle_t iph, const char *ifname,
     const char *aobjname, nvlist_t **onvl)
 {
 	ipmgmt_getaddr_arg_t	garg;
-	ipmgmt_get_rval_t	*rvalp;
-	int			err;
-	size_t			nvlsize;
-	char			*nvlbuf;
 
 	/* Populate the door_call argument structure */
 	bzero(&garg, sizeof (garg));
@@ -2609,16 +2604,7 @@ i_ipadm_get_db_addr(ipadm_handle_t iph, const char *ifname,
 	if (ifname != NULL)
 		(void) strlcpy(garg.ia_ifname, ifname, sizeof (garg.ia_ifname));
 
-	rvalp = malloc(sizeof (ipmgmt_get_rval_t));
-	err = ipadm_door_call(iph, &garg, sizeof (garg), (void **)&rvalp,
-	    sizeof (*rvalp), B_TRUE);
-	if (err == 0) {
-		nvlsize = rvalp->ir_nvlsize;
-		nvlbuf = (char *)rvalp + sizeof (ipmgmt_get_rval_t);
-		err = nvlist_unpack(nvlbuf, nvlsize, onvl, 0);
-	}
-	free(rvalp);
-	return (ipadm_errno2status(err));
+	return (i_ipadm_call_ipmgmtd(iph, (void *) &garg, sizeof (garg), onvl));
 }
 
 /*
@@ -2654,6 +2640,8 @@ ipadm_create_addr(ipadm_handle_t iph, ipadm_addrobj_t addr, uint32_t flags)
 	struct lifreq		lifr;
 	uint64_t		ifflags;
 	boolean_t		is_boot = (iph->iph_flags & IPH_IPMGMTD);
+	boolean_t		is_ipmp;
+	char			gifname[LIFGRNAMSIZ];
 
 	/* check for solaris.network.interface.config authorization */
 	if (!ipadm_check_auth())
@@ -2663,7 +2651,6 @@ ipadm_create_addr(ipadm_handle_t iph, ipadm_addrobj_t addr, uint32_t flags)
 	status = i_ipadm_validate_create_addr(iph, addr, flags);
 	if (status != IPADM_SUCCESS)
 		return (status);
-
 	/*
 	 * For Legacy case, check if an addrobj already exists for the
 	 * given logical interface name. If one does not exist,
@@ -2798,6 +2785,17 @@ ipadm_create_addr(ipadm_handle_t iph, ipadm_addrobj_t addr, uint32_t flags)
 			return (IPADM_SUCCESS);
 	}
 
+	/*
+	 * If interface is an IPMP group member, move it out of the group before
+	 * performing any operations on it.
+	 */
+	if ((is_ipmp = i_ipadm_is_under_ipmp(iph, addr->ipadm_ifname))) {
+		(void) i_ipadm_get_groupname_active(iph, addr->ipadm_ifname,
+		    gifname, sizeof (gifname));
+		(void) i_ipadm_set_groupname_active(iph, addr->ipadm_ifname,
+		    "");
+	}
+
 	/* Create the address. */
 	type = addr->ipadm_atype;
 	switch (type) {
@@ -2815,6 +2813,12 @@ ipadm_create_addr(ipadm_handle_t iph, ipadm_addrobj_t addr, uint32_t flags)
 		break;
 	}
 
+	/* Move the underlying IPMP interface back to the group */
+	if (is_ipmp) {
+		(void) i_ipadm_set_groupname_active(iph, addr->ipadm_ifname,
+		    gifname);
+	}
+
 	/*
 	 * If address was not created successfully, unplumb the interface
 	 * if it was plumbed implicitly in this function and remove the
@@ -2887,25 +2891,10 @@ i_ipadm_create_addr(ipadm_handle_t iph, ipadm_addrobj_t ipaddr, uint32_t flags)
 	 * Create a new logical interface if needed; otherwise, just
 	 * use the 0th logical interface.
 	 */
-retry:
 	if (!(iph->iph_flags & IPH_LEGACY)) {
 		status = i_ipadm_do_addif(iph, ipaddr);
 		if (status != IPADM_SUCCESS)
 			return (status);
-		/*
-		 * We don't have to set the lifnum for IPH_INIT case, because
-		 * there is no placeholder created for the address object in
-		 * this case. For IPH_LEGACY, we don't do this because the
-		 * lifnum is given by the caller and it will be set in the
-		 * end while we call the i_ipadm_addr_persist().
-		 */
-		if (!(iph->iph_flags & IPH_INIT)) {
-			status = i_ipadm_setlifnum_addrobj(iph, ipaddr);
-			if (status == IPADM_ADDROBJ_EXISTS)
-				goto retry;
-			if (status != IPADM_SUCCESS)
-				return (status);
-		}
 	}
 	i_ipadm_addrobj2lifname(ipaddr, lifr.lifr_name,
 	    sizeof (lifr.lifr_name));
@@ -2929,7 +2918,18 @@ retry:
 	}
 
 	if (flags & IPADM_OPT_UP) {
-		status = i_ipadm_set_flags(iph, lifr.lifr_name, af, IFF_UP, 0);
+		uint32_t	iff_flags = IFF_UP;
+
+		/*
+		 * Set the NOFAILOVER flag only on underlying IPMP interface
+		 * and not the IPMP group interface itself.
+		 */
+		if (i_ipadm_is_under_ipmp(iph, lifr.lifr_name) &&
+		    !i_ipadm_is_ipmp(iph, lifr.lifr_name))
+			iff_flags |= IFF_NOFAILOVER;
+
+		status = i_ipadm_set_flags(iph, lifr.lifr_name,
+		    af, iff_flags, 0);
 
 		/*
 		 * IPADM_DAD_FOUND is a soft-error for create-addr.
@@ -2964,6 +2964,7 @@ retry:
 ret:
 	if (status != IPADM_SUCCESS && !legacy)
 		(void) i_ipadm_delete_addr(iph, ipaddr);
+
 	return (status);
 }
 
@@ -2990,6 +2991,8 @@ ipadm_delete_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 	ipadm_status_t		status;
 	struct ipadm_addrobj_s	ipaddr;
 	boolean_t		release = ((flags & IPADM_OPT_RELEASE) != 0);
+	boolean_t		is_ipmp = B_FALSE;
+	char			gifname[LIFGRNAMSIZ];
 
 	/* check for solaris.network.interface.config authorization */
 	if (!ipadm_check_auth())
@@ -3028,6 +3031,19 @@ ipadm_delete_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 	 * kernel.
 	 */
 	if (ipaddr.ipadm_flags & IPMGMT_ACTIVE) {
+
+		/*
+		 * If interface is an IPMP group member, move it out of the
+		 * group before performing any operations on it.
+		 */
+		if ((is_ipmp = i_ipadm_is_under_ipmp(iph,
+		    ipaddr.ipadm_ifname))) {
+			(void) i_ipadm_get_groupname_active(iph,
+			    ipaddr.ipadm_ifname, gifname, sizeof (gifname));
+			(void) i_ipadm_set_groupname_active(iph,
+			    ipaddr.ipadm_ifname, "");
+		}
+
 		switch (ipaddr.ipadm_atype) {
 		case IPADM_ADDR_STATIC:
 			status = i_ipadm_delete_addr(iph, &ipaddr);
@@ -3056,7 +3072,7 @@ ipadm_delete_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 		if (status == IPADM_ENXIO)
 			status = IPADM_SUCCESS;
 		else if (status != IPADM_SUCCESS)
-			return (status);
+			goto out;
 	}
 
 	if (!(ipaddr.ipadm_flags & IPMGMT_PERSIST) &&
@@ -3064,9 +3080,22 @@ ipadm_delete_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 		flags &= ~IPADM_OPT_PERSIST;
 	}
 	status = i_ipadm_delete_addrobj(iph, &ipaddr, flags);
-	if (status == IPADM_NOTFOUND)
-		return (status);
-	return (IPADM_SUCCESS);
+
+	if (status != IPADM_NOTFOUND)
+		status = IPADM_SUCCESS;
+
+out:
+	/*
+	 * Move the underlying IPMP interface back to the group.
+	 * This cannot be done until the persistent configuration has been
+	 * deleted as it will otherwise cause the active configuration to be
+	 * restored.
+	 */
+	if (is_ipmp) {
+		(void) i_ipadm_set_groupname_active(iph,
+		    ipaddr.ipadm_ifname, gifname);
+	}
+	return (status);
 }
 
 /*
@@ -3568,15 +3597,18 @@ i_ipadm_updown_common(ipadm_handle_t iph, const char *aobjname,
 
 	if (!(ipaddr->ipadm_flags & IPMGMT_ACTIVE))
 		return (IPADM_OP_DISABLE_OBJ);
+
 	if ((ipadm_flags & IPADM_OPT_PERSIST) &&
 	    !(ipaddr->ipadm_flags & IPMGMT_PERSIST))
 		return (IPADM_TEMPORARY_OBJ);
+
 	if (ipaddr->ipadm_atype == IPADM_ADDR_IPV6_ADDRCONF ||
 	    (ipaddr->ipadm_atype == IPADM_ADDR_DHCP &&
 	    (ipadm_flags & IPADM_OPT_PERSIST)))
 		return (IPADM_NOTSUP);
 
 	i_ipadm_addrobj2lifname(ipaddr, lifname, sizeof (lifname));
+
 	return (i_ipadm_get_flags(iph, lifname, ipaddr->ipadm_af, ifflags));
 }
 
@@ -3803,8 +3835,13 @@ i_ipadm_validate_create_addr(ipadm_handle_t iph, ipadm_addrobj_t ipaddr,
 
 	ifname = ipaddr->ipadm_ifname;
 
-	if (i_ipadm_is_ipmp(iph, ifname) || i_ipadm_is_under_ipmp(iph, ifname))
-		return (IPADM_NOTSUP);
+	/*
+	 * Do not go further when we are under ipmp.
+	 * The interface is plumbed up and we are going to add
+	 * NOFAILOVER address to make in.mpathd happy.
+	 */
+	if (i_ipadm_is_under_ipmp(iph, ifname))
+		return (IPADM_SUCCESS);
 
 	af = ipaddr->ipadm_af;
 	af_exists = ipadm_if_enabled(iph, ifname, af);
@@ -3828,16 +3865,10 @@ i_ipadm_validate_create_addr(ipadm_handle_t iph, ipadm_addrobj_t ipaddr,
 	status = i_ipadm_if_pexists(iph, ifname, af, &p_exists);
 	if (status != IPADM_SUCCESS)
 		return (status);
+
 	if (!a_exists && p_exists)
 		return (IPADM_OP_DISABLE_OBJ);
-	if ((flags & IPADM_OPT_PERSIST) && a_exists && !p_exists) {
-		/*
-		 * If address has to be created persistently,
-		 * and the interface does not exist in the persistent
-		 * store but in active config, fail.
-		 */
-		return (IPADM_TEMPORARY_OBJ);
-	}
+
 	if (af_exists) {
 		status = i_ipadm_get_flags(iph, ifname, af, &ifflags);
 		if (status != IPADM_SUCCESS)
@@ -3957,6 +3988,8 @@ ipadm_enable_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 
 	for (nvp = nvlist_next_nvpair(addrnvl, NULL); nvp != NULL;
 	    nvp = nvlist_next_nvpair(addrnvl, nvp)) {
+		boolean_t set_init = B_FALSE;
+
 		if (nvpair_value_nvlist(nvp, &nvl) != 0)
 			continue;
 
@@ -3968,9 +4001,27 @@ ipadm_enable_addr(ipadm_handle_t iph, const char *aobjname, uint32_t flags)
 			if (status != IPADM_SUCCESS)
 				continue;
 		}
-		iph->iph_flags |= IPH_INIT;
+
+		/*
+		 * ipadm_enable_addr() is never a persistent operation. We need
+		 * to set IPH_INIT because ipmgmtd daemon does not have to write
+		 * the address to the persistent db. The address is already
+		 * available in the persistent db and we are here to re-enable
+		 * the persistent configuration.
+		 *
+		 * But we need to make sure we're not accidentally clearing an
+		 * IPH_INIT flag that was already set when we were called.
+		 */
+		if ((iph->iph_flags & IPH_INIT) == 0) {
+			iph->iph_flags |= IPH_INIT;
+			set_init = B_TRUE;
+		}
+
 		status = i_ipadm_init_addrobj(iph, nvl);
-		iph->iph_flags &= ~IPH_INIT;
+
+		if (set_init)
+			iph->iph_flags &= ~IPH_INIT;
+
 		if (status != IPADM_SUCCESS)
 			break;
 	}
diff --git a/usr/src/lib/libipadm/common/ipadm_if.c b/usr/src/lib/libipadm/common/ipadm_if.c
index c140f4ca40..73508c0d34 100644
--- a/usr/src/lib/libipadm/common/ipadm_if.c
+++ b/usr/src/lib/libipadm/common/ipadm_if.c
@@ -18,13 +18,15 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2021, Tintry by DDN. All rights reserved.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 #include <errno.h>
 #include <sys/sockio.h>
+#include <sys/list.h>
 #include <string.h>
 #include <assert.h>
 #include <unistd.h>
@@ -37,6 +39,7 @@
 #include <limits.h>
 #include <zone.h>
 #include <ipadm_ndpd.h>
+#include <ipmp_query.h>
 #include "libipadm_impl.h"
 
 static ipadm_status_t	i_ipadm_slifname_arp(char *, uint64_t, int);
@@ -45,7 +48,22 @@ static ipadm_status_t	i_ipadm_slifname(ipadm_handle_t, char *, char *,
 static ipadm_status_t	i_ipadm_create_ipmp_peer(ipadm_handle_t, char *,
 			    sa_family_t);
 static ipadm_status_t	i_ipadm_persist_if(ipadm_handle_t, const char *,
-			    sa_family_t);
+			    sa_family_t, uint32_t);
+static ipadm_status_t   i_ipadm_allocate_ifinfo(ipadm_if_info_t **);
+static ipadm_status_t	i_ipadm_get_db_if(ipadm_handle_t, const char *,
+			    nvlist_t **);
+static ipadm_status_t i_ipadm_nvl2ifinfo(nvlist_t *, ipadm_if_info_t **);
+static ipadm_status_t i_ipadm_fill_cmembers(char *, ipadm_ipmp_members_t *);
+static ipadm_status_t i_ipadm_fill_pmembers(nvlist_t *, ipadm_ipmp_members_t *);
+static ipadm_status_t i_ipadm_add_persistent_if_info(ipadm_if_info_t *,
+		    ipadm_if_info_t *);
+static void i_ipadm_free_ipmp_members(ipadm_ipmp_members_t *);
+static ipadm_status_t i_ipadm_persist_update_ipmp(ipadm_handle_t, const char *,
+	const char *,
+	ipadm_ipmp_op_t);
+static ipadm_status_t i_ipadm_update_ipmp(ipadm_handle_t, const char *,
+	const char *, uint32_t,
+	ipadm_ipmp_op_t);
 
 /*
  * Returns B_FALSE if the interface in `ifname' has at least one address that is
@@ -83,17 +101,17 @@ i_ipadm_is_if_down(char *ifname, struct ifaddrs *ifa)
  */
 static ipadm_status_t
 i_ipadm_active_if_info(ipadm_handle_t iph, const char *ifname,
-    ipadm_if_info_list_t **if_info, int64_t lifc_flags)
+    ipadm_if_info_t **if_info, int64_t lifc_flags)
 {
 	struct lifreq	*buf;
 	struct lifreq	*lifrp;
 	struct lifreq	lifrl;
-	ipadm_if_info_list_t *ifl, *last = NULL;
+	ipadm_if_info_t	*last = NULL;
 	ipadm_if_info_t	*ifp;
 	int		s;
 	int		n;
 	int		numifs;
-	ipadm_status_t	status;
+	ipadm_status_t	status = IPADM_SUCCESS;
 
 	*if_info = NULL;
 	/*
@@ -118,26 +136,23 @@ i_ipadm_active_if_info(ipadm_handle_t iph, const char *ifname,
 		 * Check if the interface already exists in our list.
 		 * If it already exists, we need to update its flags.
 		 */
-		for (ifl = *if_info; ifl != NULL; ifl = ifl->ifil_next) {
-			ifp = &ifl->ifil_ifi;
+		for (ifp = *if_info; ifp != NULL; ifp = ifp->ifi_next) {
 			if (strcmp(lifrp->lifr_name, ifp->ifi_name) == 0)
 				break;
 		}
-		if (ifl == NULL) {
-			ifl = calloc(1, sizeof (ipadm_if_info_list_t));
-			if (ifl == NULL) {
-				status = ipadm_errno2status(errno);
-				goto fail;
-			}
-			ifp = &ifl->ifil_ifi;
+		if (ifp == NULL) {
+			if ((status =
+			    i_ipadm_allocate_ifinfo(&ifp)) != IPADM_SUCCESS)
+					break;
+
 			(void) strlcpy(ifp->ifi_name, lifrp->lifr_name,
 			    sizeof (ifp->ifi_name));
-			/* Update the `ifil_next' pointer for this new node */
+			/* Update the `ifi_next' pointer for this new node */
 			if (*if_info == NULL)
-				*if_info = ifl;
+				*if_info = ifp;
 			else
-				last->ifil_next = ifl;
-			last = ifl;
+				last->ifi_next = ifp;
+			last = ifp;
 		}
 
 		/*
@@ -150,16 +165,24 @@ i_ipadm_active_if_info(ipadm_handle_t iph, const char *ifname,
 		    iph->iph_sock : iph->iph_sock6;
 		if (ioctl(s, SIOCGLIFFLAGS, (caddr_t)&lifrl) < 0)
 			continue;
+
+		/* a regular interface by default */
+		ifp->ifi_class = IPADM_IF_CLASS_REGULAR;
+
 		if (lifrl.lifr_flags & IFF_BROADCAST)
 			ifp->ifi_cflags |= IFIF_BROADCAST;
 		if (lifrl.lifr_flags & IFF_MULTICAST)
 			ifp->ifi_cflags |= IFIF_MULTICAST;
 		if (lifrl.lifr_flags & IFF_POINTOPOINT)
 			ifp->ifi_cflags |= IFIF_POINTOPOINT;
-		if (lifrl.lifr_flags & IFF_VIRTUAL)
+		if (lifrl.lifr_flags & IFF_VIRTUAL) {
 			ifp->ifi_cflags |= IFIF_VIRTUAL;
-		if (lifrl.lifr_flags & IFF_IPMP)
+			ifp->ifi_class = IPADM_IF_CLASS_VIRTUAL;
+		}
+		if (lifrl.lifr_flags & IFF_IPMP) {
 			ifp->ifi_cflags |= IFIF_IPMP;
+			ifp->ifi_class = IPADM_IF_CLASS_IPMP;
+		}
 		if (lifrl.lifr_flags & IFF_STANDBY)
 			ifp->ifi_cflags |= IFIF_STANDBY;
 		if (lifrl.lifr_flags & IFF_INACTIVE)
@@ -174,13 +197,25 @@ i_ipadm_active_if_info(ipadm_handle_t iph, const char *ifname,
 			ifp->ifi_cflags |= IFIF_IPV6;
 		if (lifrl.lifr_flags & IFF_L3PROTECT)
 			ifp->ifi_cflags |= IFIF_L3PROTECT;
+
+		/*
+		 * Retrieve active IPMP members. This may fail in in.mpathd if
+		 * the IPMP interface has just been created with no members.
+		 * Hence, ignore errors, cmembers will just be empty.
+		 */
+		if (ifp->ifi_class == IPADM_IF_CLASS_IPMP) {
+			if (ioctl(s, SIOCGLIFGROUPNAME, (caddr_t)&lifrl) == 0) {
+				(void) i_ipadm_fill_cmembers(
+				    lifrl.lifr_groupname,
+				    &ifp->ifi_ipmp_cmembers);
+			}
+		}
 	}
 	free(buf);
-	return (IPADM_SUCCESS);
-fail:
-	free(buf);
-	ipadm_free_if_info(*if_info);
-	*if_info = NULL;
+	if (status != IPADM_SUCCESS) {
+		ipadm_free_if_info(*if_info);
+		*if_info = NULL;
+	}
 	return (status);
 }
 
@@ -191,54 +226,153 @@ fail:
  */
 static ipadm_status_t
 i_ipadm_persist_if_info(ipadm_handle_t iph, const char *ifname,
-    ipadm_if_info_list_t **if_info)
+    ipadm_if_info_t **if_info)
 {
-	ipadm_status_t		status = IPADM_SUCCESS;
-	ipmgmt_getif_arg_t	getif;
-	ipmgmt_getif_rval_t	*rvalp;
-	ipadm_if_info_t		*ifp;
-	ipadm_if_info_list_t	*curr, *prev = NULL;
-	int			i = 0, err = 0;
-
-	bzero(&getif, sizeof (getif));
-	if (ifname != NULL)
-		(void) strlcpy(getif.ia_ifname, ifname, LIFNAMSIZ);
-	getif.ia_cmd = IPMGMT_CMD_GETIF;
+	ipadm_status_t	status = IPADM_SUCCESS;
+	nvlist_t	*ifs_info_nvl;
 
 	*if_info = NULL;
 
-	if ((rvalp = malloc(sizeof (ipmgmt_getif_rval_t))) == NULL)
-		return (ipadm_errno2status(errno));
-	err = ipadm_door_call(iph, &getif, sizeof (getif), (void **)&rvalp,
-	    sizeof (*rvalp), B_TRUE);
-	if (err == ENOENT) {
-		free(rvalp);
-		if (ifname != NULL)
-			return (ipadm_errno2status(err));
-		return (IPADM_SUCCESS);
-	} else if (err != 0) {
-		free(rvalp);
-		return (ipadm_errno2status(err));
-	}
+	if ((status = i_ipadm_get_db_if(iph,
+	    ifname, &ifs_info_nvl)) != IPADM_SUCCESS)
+		return (status);
 
-	ifp = rvalp->ir_ifinfo;
-	for (i = 0; i < rvalp->ir_ifcnt; i++) {
-		ifp = rvalp->ir_ifinfo + i;
-		if ((curr = malloc(sizeof (*curr))) == NULL) {
-			status = ipadm_errno2status(errno);
-			ipadm_free_if_info(prev);
+	assert(ifs_info_nvl != NULL);
+
+	return (i_ipadm_nvl2ifinfo(ifs_info_nvl, if_info));
+}
+
+static ipadm_status_t
+i_ipadm_nvl2ifinfo(nvlist_t *ifs_info_nvl, ipadm_if_info_t **if_info)
+{
+	ipadm_if_info_t *ific = NULL, *ifil = NULL;
+	nvlist_t	*if_info_nvl;
+	nvpair_t	*nvp;
+	char		*strval;
+	ipadm_status_t	status = IPADM_SUCCESS;
+	uint16_t	*families;
+	uint_t		nelem = 0;
+
+	for (nvp = nvlist_next_nvpair(ifs_info_nvl, NULL); nvp != NULL;
+	    nvp = nvlist_next_nvpair(ifs_info_nvl, nvp)) {
+		if (nvpair_value_nvlist(nvp, &if_info_nvl) != 0)
+			continue;
+
+		status = i_ipadm_allocate_ifinfo(&ific);
+		if (status != IPADM_SUCCESS) {
+			ipadm_free_if_info(*if_info);
 			break;
 		}
-		(void) bcopy(ifp, &curr->ifil_ifi, sizeof (*ifp));
-		curr->ifil_next = prev;
-		prev = curr;
+		if (nvlist_lookup_string(if_info_nvl, IPADM_NVP_IFNAME,
+		    &strval) != 0) {
+			ipadm_free_if_info(ific);
+			ific = NULL;
+			continue;
+		}
+		(void) strlcpy(ific->ifi_name, strval,
+		    sizeof (ific->ifi_name));
+
+		if (nvlist_lookup_uint16_array(if_info_nvl,
+		    IPADM_NVP_FAMILIES, &families, &nelem) == 0) {
+			while (nelem-- > 0) {
+				if (families[nelem] == AF_INET)
+					ific->ifi_pflags |= IFIF_IPV4;
+				else if (families[nelem] == AF_INET6)
+					ific->ifi_pflags |= IFIF_IPV6;
+			}
+		}
+
+		if (nvlist_lookup_string(if_info_nvl,
+		    IPADM_NVP_IFCLASS, &strval) == 0)
+			ific->ifi_class = atoi(strval);
+		else
+			ific->ifi_class = IPADM_IF_CLASS_REGULAR;
+
+		if (ific->ifi_class == IPADM_IF_CLASS_IPMP)
+			/* do not expect any failures there */
+			(void) i_ipadm_fill_pmembers(if_info_nvl,
+			    &ific->ifi_ipmp_pmembers);
+
+		if (*if_info == NULL)
+			*if_info = ific;
+		else
+			ifil->ifi_next = ific;
+		ifil = ific;
 	}
-	*if_info = curr;
-	free(rvalp);
+
+	nvlist_free(ifs_info_nvl);
 	return (status);
 }
 
 /*
+ * Fill the ipadm_if_info_t->ifi_ipmp_pmembers by info from
+ * ipadm DB
+ */
+static ipadm_status_t
+i_ipadm_fill_pmembers(nvlist_t *if_info_nvl, ipadm_ipmp_members_t *pmembers)
+{
+	uint_t	nelem = 0;
+	char	**members;
+	ipadm_ipmp_member_t *ipmp_member;
+
+	if (nvlist_lookup_string_array(if_info_nvl, IPADM_NVP_MIFNAMES,
+	    &members, &nelem) != 0)
+		return (IPADM_SUCCESS);
+
+	while (nelem-- > 0) {
+		if ((ipmp_member = calloc(1,
+		    sizeof (ipadm_ipmp_member_t))) == NULL)
+			return (ipadm_errno2status(errno));
+
+		(void) strlcpy(ipmp_member->if_name, members[nelem],
+		    sizeof (ipmp_member->if_name));
+		list_insert_tail(pmembers, ipmp_member);
+	}
+	return (IPADM_SUCCESS);
+}
+
+/*
+ * Fill the ipadm_if_info_t->ifi_ipmp_cmembers by info from
+ * kernel (libipmp is used to retrieve the required info)
+ */
+static ipadm_status_t
+i_ipadm_fill_cmembers(char *grname, ipadm_ipmp_members_t *cmembers)
+{
+	ipmp_handle_t ipmp_handle;
+	ipmp_groupinfo_t *grinfo;
+	ipmp_iflist_t *iflistp;
+	ipadm_ipmp_member_t *ipmp_member;
+	ipadm_status_t ipadm_status = IPADM_SUCCESS;
+	int i;
+
+	if (ipmp_open(&ipmp_handle) != IPMP_SUCCESS)
+		return (IPADM_FAILURE);
+
+	if (ipmp_getgroupinfo(ipmp_handle, grname, &grinfo) != IPMP_SUCCESS) {
+		ipadm_status = IPADM_FAILURE;
+		goto fail2;
+	}
+
+	iflistp = grinfo->gr_iflistp;
+	for (i = 0; i < iflistp->il_nif; i++) {
+		if ((ipmp_member = calloc(1,
+		    sizeof (ipadm_ipmp_member_t))) == NULL) {
+			ipadm_status = ipadm_errno2status(errno);
+			goto fail1;
+		}
+		(void) strlcpy(ipmp_member->if_name, iflistp->il_ifs[i],
+		    sizeof (ipmp_member->if_name));
+		list_insert_tail(cmembers, ipmp_member);
+	}
+
+fail1:
+	ipmp_freegroupinfo(grinfo);
+fail2:
+	ipmp_close(ipmp_handle);
+	return (ipadm_status);
+}
+
+/*
  * Collects information for `ifname' if one is specified from both
  * active and persistent config in `if_info'. If no `ifname' is specified,
  * this returns all the interfaces in active and persistent config in
@@ -246,16 +380,14 @@ i_ipadm_persist_if_info(ipadm_handle_t iph, const char *ifname,
  */
 ipadm_status_t
 i_ipadm_get_all_if_info(ipadm_handle_t iph, const char *ifname,
-    ipadm_if_info_list_t **if_info, int64_t lifc_flags)
+    ipadm_if_info_t **if_info, int64_t lifc_flags)
 {
 	ipadm_status_t	status;
-	ipadm_if_info_list_t *aifinfo = NULL;
-	ipadm_if_info_list_t *pifinfo = NULL;
-	ipadm_if_info_list_t *last = NULL;
-	ipadm_if_info_list_t *aifl;
-	ipadm_if_info_list_t *pifl;
+	ipadm_if_info_t	*aifinfo = NULL;
+	ipadm_if_info_t	*pifinfo = NULL;
 	ipadm_if_info_t	*aifp;
 	ipadm_if_info_t	*pifp;
+	ipadm_if_info_t	*last = NULL;
 	struct ifaddrs	*ifa;
 	struct ifaddrs	*ifap;
 
@@ -275,9 +407,7 @@ retry:
 			status = ipadm_errno2status(errno);
 			goto fail;
 		}
-		for (aifl = aifinfo; aifl != NULL; aifl = aifl->ifil_next) {
-			aifp = &aifl->ifil_ifi;
-
+		for (aifp = aifinfo; aifp != NULL; aifp = aifp->ifi_next) {
 			/*
 			 * Find the `ifaddrs' structure from `ifa'
 			 * for this interface. We need the IFF_* flags
@@ -307,8 +437,8 @@ retry:
 				aifp->ifi_state = IFIS_DOWN;
 			else
 				aifp->ifi_state = IFIS_OK;
-			if (aifl->ifil_next == NULL)
-				last = aifl;
+			if (aifp->ifi_next == NULL)
+				last = aifp;
 		}
 		freeifaddrs(ifa);
 	}
@@ -322,37 +452,55 @@ retry:
 	}
 	if (status != IPADM_SUCCESS)
 		goto fail;
+
 	/*
-	 * If a persistent interface is also found in `aifinfo', update
+	 * Process the persistent interface information.
+	 *
+	 * First try to get the persistent "standby" property, as that isn't
+	 * retrieved by i_ipadm_persist_if_info().
+	 *
+	 * Next, if a persistent interface is also found in `aifinfo', update
 	 * its entry in `aifinfo' with the persistent information from
 	 * `pifinfo'. If an interface is found in `pifinfo', but not in
 	 * `aifinfo', it means that this interface was disabled. We should
 	 * add this interface to `aifinfo' and set it state to IFIF_DISABLED.
 	 */
-	for (pifl = pifinfo; pifl != NULL; pifl = pifl->ifil_next) {
-		pifp = &pifl->ifil_ifi;
-		for (aifl = aifinfo; aifl != NULL; aifl = aifl->ifil_next) {
-			aifp = &aifl->ifil_ifi;
+	for (pifp = pifinfo; pifp != NULL; pifp = pifp->ifi_next) {
+		char buf[10] = "";
+		uint_t bufsize = sizeof (buf);
+
+		status = ipadm_get_ifprop(iph, pifp->ifi_name, "standby", buf,
+		    &bufsize, MOD_PROTO_IP, IPADM_OPT_PERSIST);
+
+		if (status == IPADM_SUCCESS && strcmp(buf, "on") == 0)
+			pifp->ifi_pflags |= IFIF_STANDBY;
+
+		for (aifp = aifinfo; aifp != NULL; aifp = aifp->ifi_next) {
 			if (strcmp(aifp->ifi_name, pifp->ifi_name) == 0) {
-				aifp->ifi_pflags = pifp->ifi_pflags;
 				break;
 			}
 		}
-		if (aifl == NULL) {
-			aifl = malloc(sizeof (ipadm_if_info_list_t));
-			if (aifl == NULL) {
-				status = ipadm_errno2status(errno);
+
+		if (aifp == NULL) {
+			if ((status =
+			    i_ipadm_allocate_ifinfo(&aifp)) != IPADM_SUCCESS)
 				goto fail;
-			}
-			*aifl = *pifl;
-			aifl->ifil_next = NULL;
-			aifl->ifil_ifi.ifi_state = IFIS_DISABLED;
+
+			(void) strlcpy(aifp->ifi_name, pifp->ifi_name,
+			    sizeof (aifp->ifi_name));
+
+			aifp->ifi_next = NULL;
+			aifp->ifi_state = IFIS_DISABLED;
 			if (last != NULL)
-				last->ifil_next = aifl;
+				last->ifi_next = aifp;
 			else
-				aifinfo = aifl;
-			last = aifl;
+				aifinfo = aifp;
+			last = aifp;
 		}
+
+		if ((status = i_ipadm_add_persistent_if_info(aifp,
+		    pifp)) != IPADM_SUCCESS)
+			goto fail;
 	}
 	*if_info = aifinfo;
 	ipadm_free_if_info(pifinfo);
@@ -364,6 +512,75 @@ fail:
 	return (status);
 }
 
+/*
+ * Updates active if_info by data from persistent if_info
+ */
+static ipadm_status_t
+i_ipadm_add_persistent_if_info(ipadm_if_info_t *aifp, ipadm_if_info_t *pifp)
+{
+	ipadm_ipmp_member_t *pp_ipmp_member, *ap_ipmp_member;
+
+	ipadm_ipmp_members_t *apmembers = &aifp->ifi_ipmp_pmembers;
+	ipadm_ipmp_members_t *ppmembers = &pifp->ifi_ipmp_pmembers;
+
+	aifp->ifi_pflags = pifp->ifi_pflags;
+	aifp->ifi_class = pifp->ifi_class;
+
+	for (pp_ipmp_member = list_head(ppmembers); pp_ipmp_member;
+	    pp_ipmp_member = list_next(ppmembers, pp_ipmp_member)) {
+		if ((ap_ipmp_member = calloc(1,
+		    sizeof (ipadm_ipmp_member_t))) == NULL)
+			return (ipadm_errno2status(errno));
+
+		(void) strlcpy(ap_ipmp_member->if_name,
+		    pp_ipmp_member->if_name,
+		    sizeof (ap_ipmp_member->if_name));
+
+		list_insert_tail(apmembers, ap_ipmp_member);
+	}
+	return (IPADM_SUCCESS);
+}
+
+static ipadm_status_t
+i_ipadm_allocate_ifinfo(ipadm_if_info_t **if_info)
+{
+	*if_info = calloc(1, sizeof (ipadm_if_info_t));
+	if (*if_info == NULL)
+		return (ipadm_errno2status(errno));
+
+	/* List of active (current) members */
+	list_create(&((*if_info)->ifi_ipmp_cmembers),
+	    sizeof (ipadm_ipmp_member_t),
+	    offsetof(ipadm_ipmp_member_t, node));
+
+	/* List of persistent members */
+	list_create(&((*if_info)->ifi_ipmp_pmembers),
+	    sizeof (ipadm_ipmp_member_t),
+	    offsetof(ipadm_ipmp_member_t, node));
+
+	return (IPADM_SUCCESS);
+}
+
+/*
+ * Reads all the interface lines from the persistent DB into the nvlist `onvl',
+ * when `ifname' is NULL.
+ * If an `ifname' is specified, then the interface line corresponding to
+ * that name will be returned.
+ */
+static ipadm_status_t
+i_ipadm_get_db_if(ipadm_handle_t iph, const char *ifname, nvlist_t **onvl)
+{
+	ipmgmt_getif_arg_t	garg;
+
+	/* Populate the door_call argument structure */
+	bzero(&garg, sizeof (garg));
+	garg.ia_cmd = IPMGMT_CMD_GETIF;
+	if (ifname != NULL)
+		(void) strlcpy(garg.ia_ifname, ifname, sizeof (garg.ia_ifname));
+
+	return (i_ipadm_call_ipmgmtd(iph, (void *) &garg, sizeof (garg), onvl));
+}
+
 int
 i_ipadm_get_lnum(const char *ifname)
 {
@@ -385,7 +602,7 @@ ipadm_status_t
 i_ipadm_if_pexists(ipadm_handle_t iph, const char *ifname, sa_family_t af,
     boolean_t *exists)
 {
-	ipadm_if_info_list_t	*ifinfo;
+	ipadm_if_info_t	*ifinfo;
 	ipadm_status_t	status;
 
 	/*
@@ -400,10 +617,10 @@ i_ipadm_if_pexists(ipadm_handle_t iph, const char *ifname, sa_family_t af,
 	status = i_ipadm_persist_if_info(iph, ifname, &ifinfo);
 	if (status == IPADM_SUCCESS) {
 		*exists = ((af == AF_INET &&
-		    (ifinfo->ifil_ifi.ifi_pflags & IFIF_IPV4)) ||
+		    (ifinfo->ifi_pflags & IFIF_IPV4)) ||
 		    (af == AF_INET6 &&
-		    (ifinfo->ifil_ifi.ifi_pflags & IFIF_IPV6)));
-		free(ifinfo);
+		    (ifinfo->ifi_pflags & IFIF_IPV6)));
+		ipadm_free_if_info(ifinfo);
 	} else if (status == IPADM_NOTFOUND) {
 		status = IPADM_SUCCESS;
 		*exists = B_FALSE;
@@ -742,7 +959,8 @@ i_ipadm_plumb_if(ipadm_handle_t iph, char *ifname, sa_family_t af,
 			if (ioctl(sock, SIOCSLIFADDR, (caddr_t)&lifr) < 0)
 				return (ipadm_errno2status(errno));
 			if (is_persistent) {
-				status = i_ipadm_persist_if(iph, ifname, af);
+				status = i_ipadm_persist_if(iph,
+				    ifname, af, ipadm_flags);
 				if (status != IPADM_SUCCESS) {
 					(void) i_ipadm_delete_if(iph, ifname,
 					    af, IPADM_OPT_ACTIVE);
@@ -923,7 +1141,8 @@ done:
 		 * interface in persistent DB.
 		 */
 		if (is_persistent) {
-			status = i_ipadm_persist_if(iph, newif, af);
+			status = i_ipadm_persist_if(iph,
+			    newif, af, ipadm_flags);
 			if (status != IPADM_SUCCESS) {
 				(void) i_ipadm_delete_if(iph, newif, af,
 				    IPADM_OPT_ACTIVE);
@@ -1156,13 +1375,19 @@ done:
  * persistent DB.
  */
 static ipadm_status_t
-i_ipadm_persist_if(ipadm_handle_t iph, const char *ifname, sa_family_t af)
+i_ipadm_persist_if(ipadm_handle_t iph, const char *ifname, sa_family_t af,
+    uint32_t ipadm_flags)
 {
 	ipmgmt_if_arg_t		ifarg;
 	int			err;
 
 	(void) strlcpy(ifarg.ia_ifname, ifname, sizeof (ifarg.ia_ifname));
 	ifarg.ia_family = af;
+	if (ipadm_flags & IPADM_OPT_IPMP)
+		ifarg.ia_ifclass = IPADM_IF_CLASS_IPMP;
+	else
+		ifarg.ia_ifclass = IPADM_IF_CLASS_REGULAR;
+
 	ifarg.ia_cmd = IPMGMT_CMD_SETIF;
 	ifarg.ia_flags = IPMGMT_PERSIST;
 	err = ipadm_door_call(iph, &ifarg, sizeof (ifarg), NULL, 0, B_FALSE);
@@ -1355,6 +1580,109 @@ ipadm_create_if(ipadm_handle_t iph, char *ifname, sa_family_t af,
 	return (IPADM_SUCCESS);
 }
 
+ipadm_status_t
+ipadm_add_ipmp_member(ipadm_handle_t iph, const char *gifname,
+    const char *mifname, uint32_t ipadm_flags)
+{
+	return (i_ipadm_update_ipmp(iph, gifname, mifname,
+	    ipadm_flags, IPADM_ADD_IPMP));
+}
+
+ipadm_status_t
+ipadm_remove_ipmp_member(ipadm_handle_t iph, const char *gifname,
+    const char *mifname, uint32_t ipadm_flags)
+{
+	return (i_ipadm_update_ipmp(iph, gifname, mifname,
+	    ipadm_flags, IPADM_REMOVE_IPMP));
+}
+
+/*
+ * Updates active IPMP configuration according to the specified
+ * command. It also persists the configuration if IPADM_OPT_PERSIST
+ * is set in `ipadm_flags'.
+ */
+static ipadm_status_t
+i_ipadm_update_ipmp(ipadm_handle_t iph, const char *gifname,
+    const char *mifname, uint32_t ipadm_flags, ipadm_ipmp_op_t op)
+{
+	ipadm_status_t status;
+	char	groupname1[LIFGRNAMSIZ];
+	char	groupname2[LIFGRNAMSIZ];
+
+	/* Check for the required authorization */
+	if (!ipadm_check_auth())
+		return (IPADM_EAUTH);
+
+	if (!(ipadm_flags & IPADM_OPT_ACTIVE) ||
+	    gifname == NULL || mifname == NULL)
+		return (IPADM_INVALID_ARG);
+
+	if (!ipadm_if_enabled(iph, gifname, AF_UNSPEC) ||
+	    !ipadm_if_enabled(iph, mifname, AF_UNSPEC))
+		return (IPADM_OP_DISABLE_OBJ);
+
+	if (!i_ipadm_is_ipmp(iph, gifname))
+		return (IPADM_INVALID_ARG);
+
+	if (op == IPADM_ADD_IPMP && i_ipadm_is_under_ipmp(iph, mifname))
+		return (IPADM_IF_INUSE);
+
+	if ((status = i_ipadm_get_groupname_active(iph, gifname,
+	    groupname2, sizeof (groupname2))) != IPADM_SUCCESS)
+		return (status);
+
+	if (op == IPADM_REMOVE_IPMP) {
+		if ((status = i_ipadm_get_groupname_active(iph, mifname,
+		    groupname1, sizeof (groupname1))) != IPADM_SUCCESS)
+			return (status);
+
+		if (groupname1[0] == '\0' ||
+		    strcmp(groupname1, groupname2) != 0)
+			return (IPADM_INVALID_ARG);
+
+		groupname2[0] = '\0';
+	}
+
+	if ((ipadm_flags & IPADM_OPT_PERSIST) &&
+	    (status = i_ipadm_persist_update_ipmp(iph, gifname,
+	    mifname, op)) != IPADM_SUCCESS)
+		return (status);
+
+	return (i_ipadm_set_groupname_active(iph, mifname, groupname2));
+}
+
+/*
+ * Call the ipmgmtd to update the IPMP configuration in ipadm DB.
+ * After this call the DB will know that mifname is under gifname and
+ * gifname has a member, which name is mifname.
+ */
+static ipadm_status_t
+i_ipadm_persist_update_ipmp(ipadm_handle_t iph, const char *gifname,
+    const char *mifname, ipadm_ipmp_op_t op)
+{
+	ipmgmt_ipmp_update_arg_t args;
+	int err;
+
+	assert(op == IPADM_ADD_IPMP || op == IPADM_REMOVE_IPMP);
+
+	bzero(&args, sizeof (ipmgmt_ipmp_update_arg_t));
+
+	args.ia_cmd = IPMGMT_CMD_IPMP_UPDATE;
+
+	(void) strlcpy(args.ia_gifname, gifname, sizeof (args.ia_gifname));
+	(void) strlcpy(args.ia_mifname, mifname, sizeof (args.ia_mifname));
+
+	if (op == IPADM_ADD_IPMP)
+		args.ia_flags = IPMGMT_APPEND;
+	else
+		args.ia_flags = IPMGMT_REMOVE;
+
+	args.ia_flags |= IPMGMT_PERSIST;
+
+	err = ipadm_door_call(iph, &args, sizeof (args), NULL, 0, B_FALSE);
+	return (ipadm_errno2status(err));
+}
+
 /*
  * Deletes the interface in `ifname'. Removes both IPv4 and IPv6 interfaces
  * when `af' = AF_UNSPEC.
@@ -1426,7 +1754,7 @@ ipadm_delete_if(ipadm_handle_t iph, const char *ifname, sa_family_t af,
  */
 ipadm_status_t
 ipadm_if_info(ipadm_handle_t iph, const char *ifname,
-    ipadm_if_info_list_t **if_info, uint32_t flags, int64_t lifc_flags)
+    ipadm_if_info_t **if_info, uint32_t flags, int64_t lifc_flags)
 {
 	ipadm_status_t	status;
 	ifspec_t	ifsp;
@@ -1452,16 +1780,29 @@ ipadm_if_info(ipadm_handle_t iph, const char *ifname,
  * Frees the linked list allocated by ipadm_if_info().
  */
 void
-ipadm_free_if_info(ipadm_if_info_list_t *ifinfo)
+ipadm_free_if_info(ipadm_if_info_t *ifinfo)
 {
-	ipadm_if_info_list_t	*ifinfo_next;
+	ipadm_if_info_t	*ifinfo_next;
 
 	for (; ifinfo != NULL; ifinfo = ifinfo_next) {
-		ifinfo_next = ifinfo->ifil_next;
+		ifinfo_next = ifinfo->ifi_next;
+		i_ipadm_free_ipmp_members(&ifinfo->ifi_ipmp_cmembers);
+		i_ipadm_free_ipmp_members(&ifinfo->ifi_ipmp_pmembers);
 		free(ifinfo);
 	}
 }
 
+static void
+i_ipadm_free_ipmp_members(ipadm_ipmp_members_t *ipmp_members)
+{
+	ipadm_ipmp_member_t *ipmp_member;
+
+	while ((ipmp_member = list_remove_head(ipmp_members)) != NULL)
+		free(ipmp_member);
+
+	list_destroy(ipmp_members);
+}
+
 /*
  * Re-enable the interface `ifname' based on the saved configuration
  * for `ifname'.
@@ -1469,6 +1810,7 @@ ipadm_free_if_info(ipadm_if_info_list_t *ifinfo)
 ipadm_status_t
 ipadm_enable_if(ipadm_handle_t iph, const char *ifname, uint32_t flags)
 {
+	boolean_t	set_init = B_FALSE;
 	nvlist_t	*ifnvl;
 	ipadm_status_t	status;
 	ifspec_t	ifsp;
@@ -1489,9 +1831,9 @@ ipadm_enable_if(ipadm_handle_t iph, const char *ifname, uint32_t flags)
 	 * Return early by checking if the interface is already enabled.
 	 */
 	if (ipadm_if_enabled(iph, ifname, AF_INET) &&
-	    ipadm_if_enabled(iph, ifname, AF_INET6)) {
+	    ipadm_if_enabled(iph, ifname, AF_INET6))
 		return (IPADM_IF_EXISTS);
-	}
+
 	/*
 	 * Enable the interface and restore all its interface properties
 	 * and address objects.
@@ -1504,13 +1846,23 @@ ipadm_enable_if(ipadm_handle_t iph, const char *ifname, uint32_t flags)
 	/*
 	 * ipadm_enable_if() does exactly what ipadm_init_ifs() does,
 	 * but only for one interface. We need to set IPH_INIT because
-	 * ipmgmtd daemon does not have to write the interface to persistent
-	 * db. The interface is already available in persistent db
-	 * and we are here to re-enable the persistent configuration.
+	 * ipmgmtd daemon does not have to write the interface to the
+	 * persistent db. The interface is already available in the
+	 * persistent db and we are here to re-enable the persistent
+	 * configuration.
+	 *
+	 * But we need to make sure we're not accidentally clearing an
+	 * IPH_INIT flag that was already set when we were called.
 	 */
-	iph->iph_flags |= IPH_INIT;
+	if ((iph->iph_flags & IPH_INIT) == 0) {
+		iph->iph_flags |= IPH_INIT;
+		set_init = B_TRUE;
+	}
+
 	status = i_ipadm_init_ifobj(iph, ifname, ifnvl);
-	iph->iph_flags &= ~IPH_INIT;
+
+	if (set_init)
+		iph->iph_flags &= ~IPH_INIT;
 
 	nvlist_free(ifnvl);
 	return (status);
@@ -1558,10 +1910,13 @@ ipadm_disable_if(ipadm_handle_t iph, const char *ifname, uint32_t flags)
 }
 
 /*
- * This workaround is until libipadm supports IPMP and is required whenever an
- * interface is moved into an IPMP group. Since libipadm doesn't support IPMP
- * yet, we will have to update the daemon's in-memory mapping of
- * `aobjname' to 'lifnum'.
+ * FIXME Remove this when ifconfig(1M) is updated to use IPMP support
+ * in libipadm.
+ */
+/*
+ * This workaround is required by ifconfig(1M) whenever an
+ * interface is moved into an IPMP group to update the daemon's
+ * in-memory mapping of `aobjname' to 'lifnum'.
  *
  * For `IPMGMT_ACTIVE' case, i_ipadm_delete_ifobj() would only fail if
  * door_call(3C) fails. Also, there is no use in returning error because
@@ -1573,3 +1928,132 @@ ipadm_if_move(ipadm_handle_t iph, const char *ifname)
 	(void) i_ipadm_delete_ifobj(iph, ifname, AF_INET, B_FALSE);
 	(void) i_ipadm_delete_ifobj(iph, ifname, AF_INET6, B_FALSE);
 }
+
+ipadm_status_t
+i_ipadm_set_groupname_active(ipadm_handle_t iph, const char *ifname,
+    const char *groupname)
+{
+	struct lifreq   lifr;
+	ipadm_addr_info_t *addrinfo, *ia;
+	ipadm_status_t	status = IPADM_SUCCESS;
+
+	(void) memset(&lifr, 0, sizeof (lifr));
+
+	(void) strlcpy(lifr.lifr_name, ifname, sizeof (lifr.lifr_name));
+	(void) strlcpy(lifr.lifr_groupname, groupname,
+	    sizeof (lifr.lifr_groupname));
+
+	/* Disable all addresses on the interface */
+	(void) i_ipadm_active_addr_info(iph, ifname, &addrinfo,
+	    IPADM_OPT_ACTIVE | IPADM_OPT_ZEROADDR, IFF_UP | IFF_DUPLICATE);
+
+	for (ia = addrinfo; ia != NULL; ia = IA_NEXT(ia)) {
+		if (strlen(ia->ia_aobjname) > 0) {
+			(void) ipadm_disable_addr(iph, ia->ia_aobjname, 0);
+		} else {
+			/*
+			 * There's an address on this interfaces with no
+			 * corresponding addrobj. Just clear IFF_UP.
+			 */
+			(void) i_ipadm_set_flags(iph, ifname,
+			    addrinfo->ia_ifa.ifa_addr->sa_family, 0, IFF_UP);
+		}
+	}
+
+	if (ioctl(iph->iph_sock, SIOCSLIFGROUPNAME, (caddr_t)&lifr) == -1 &&
+	    ioctl(iph->iph_sock6, SIOCSLIFGROUPNAME, (caddr_t)&lifr) == -1)
+		status = ipadm_errno2status(errno);
+
+	/* Enable all addresses on the interface */
+	for (ia = addrinfo; ia != NULL; ia = IA_NEXT(ia)) {
+		if (strlen(ia->ia_aobjname) > 0) {
+			(void) ipadm_enable_addr(iph, ia->ia_aobjname, 0);
+		} else {
+			/*
+			 * There's an address on this interfaces with no
+			 * corresponding addrobj. Just set IFF_UP.
+			 */
+			(void) i_ipadm_set_flags(iph, ifname,
+			    addrinfo->ia_ifa.ifa_addr->sa_family, IFF_UP, 0);
+		}
+	}
+
+	if (status == IPADM_SUCCESS) {
+		if (groupname[0] == '\0') {
+			/*
+			 * If interface was removed from IPMP group, unset the
+			 * DEPRECATED and NOFAILOVER flags.
+			 */
+			(void) i_ipadm_set_flags(iph, ifname, AF_INET, 0,
+			    IFF_DEPRECATED | IFF_NOFAILOVER);
+			(void) i_ipadm_set_flags(iph, ifname, AF_INET6, 0,
+			    IFF_DEPRECATED | IFF_NOFAILOVER);
+		} else if (addrinfo == NULL) {
+			/*
+			 * If interface was added to IPMP group and there are no
+			 * active addresses, explicitly bring it up to be used
+			 * for link-based IPMP configuration.
+			 */
+			(void) i_ipadm_set_flags(iph, ifname, AF_INET,
+			    IFF_UP, 0);
+			(void) i_ipadm_set_flags(iph, ifname, AF_INET6,
+			    IFF_UP, 0);
+		}
+	}
+
+	ipadm_free_addr_info(addrinfo);
+
+	return (status);
+}
+
+ipadm_status_t
+i_ipadm_get_groupname_active(ipadm_handle_t iph, const char *ifname,
+    char *groupname, size_t size)
+{
+	struct lifreq   lifr;
+
+	(void) memset(&lifr, 0, sizeof (lifr));
+
+	(void) strlcpy(lifr.lifr_name, ifname, sizeof (lifr.lifr_name));
+
+	if (ioctl(iph->iph_sock, SIOCGLIFGROUPNAME, (caddr_t)&lifr) == -1 &&
+	    ioctl(iph->iph_sock6, SIOCGLIFGROUPNAME, (caddr_t)&lifr) == -1)
+		return (ipadm_errno2status(errno));
+
+	(void) strlcpy(groupname, lifr.lifr_groupname, size);
+
+	return (IPADM_SUCCESS);
+}
+
+/*
+ * Returns B_TRUE if `ifname' represents an IPMP underlying interface.
+ */
+boolean_t
+i_ipadm_is_under_ipmp(ipadm_handle_t iph, const char *ifname)
+{
+
+	char	groupname[LIFGRNAMSIZ];
+
+	if (i_ipadm_get_groupname_active(iph, ifname, groupname,
+	    sizeof (groupname)) != IPADM_SUCCESS ||
+	    groupname[0] == '\0' ||
+	    strcmp(ifname, groupname) == 0)
+		return (B_FALSE);
+
+	return (B_TRUE);
+}
+
+/*
+ * Returns B_TRUE if `ifname' represents an IPMP group interface.
+ */
+boolean_t
+i_ipadm_is_ipmp(ipadm_handle_t iph, const char *ifname)
+{
+	uint64_t flags;
+
+	if (i_ipadm_get_flags(iph, ifname, AF_INET, &flags) != IPADM_SUCCESS &&
+	    i_ipadm_get_flags(iph, ifname, AF_INET6, &flags) != IPADM_SUCCESS)
+		return (B_FALSE);
+
+	return ((flags & IFF_IPMP) != 0);
+}
diff --git a/usr/src/lib/libipadm/common/ipadm_ipmgmt.h b/usr/src/lib/libipadm/common/ipadm_ipmgmt.h
index 4c6c66de96..4a5c408b57 100644
--- a/usr/src/lib/libipadm/common/ipadm_ipmgmt.h
+++ b/usr/src/lib/libipadm/common/ipadm_ipmgmt.h
@@ -58,6 +58,10 @@ extern "C" {
  */
 #define	IPADM_NVP_PROTONAME	"_protocol"	/* protocol name */
 #define	IPADM_NVP_IFNAME	"_ifname"	/* interface name */
+#define	IPADM_NVP_IFCLASS	"_ifclass"	/* interface class */
+#define	IPADM_NVP_FAMILIES	"_families"	/* interface families */
+#define	IPADM_NVP_GIFNAME	"_gifname"	/* IPMP group interface name */
+#define	IPADM_NVP_MIFNAMES	"_mifnames"	/* IPMP group members */
 #define	IPADM_NVP_AOBJNAME	"_aobjname"	/* addrobj name */
 #define	IPADM_NVP_FAMILY	"_family"	/* address family */
 #define	IPADM_NVP_IPV4ADDR	"_ipv4addr"	/* name of IPv4 addr nvlist */
@@ -109,6 +113,38 @@ typedef	struct	ipadm_dbwrite_cbarg_s {
 #define	IPMGMT_DOOR		"/etc/svc/volatile/ipadm/ipmgmt_door"
 #define	MAXPROTONAMELEN		32
 
+/*
+ * ia_flags used inside the arguments for interface/address commands
+ *
+ *	- APPEND updates the multi-valued ipadm DB entry with a new value
+ *	- REMOVE updates the multi-valued ipadm DB entry by removing a value
+ *	- ACTIVE updates the running configuration
+ *	- PERSIST updates the permanent data store
+ *	- INIT indicates that operation being performed is under init context
+ *	- PROPS_ONLY indicates the update changes the running configuration of
+ *	  "props" data on the interface/address object. The props are cached
+ *	  there on the parent, so a PROPS_ONLY change does not affect the
+ *	  ACTIVE/PERSIST state of the parent.
+ *
+ * These two flags are used by ipmgmt_db_update_if function,
+ * because it can be used to update more that one DB line
+ * and we need to be sure that we finished all operations,
+ * after the operation has finished the related flag is cleared
+ *
+ *  - UPDATE_IF - used when we need to update IPADM_NVP_FAMILIES and
+ *                IPADM_NVP_MIFNAMES fields
+ *  - UPDATE_IPMP - used when we need to update IPADM_NVP_GIFNAME
+ */
+#define	IPMGMT_APPEND	0x00000001
+#define	IPMGMT_REMOVE	0x00000002
+#define	IPMGMT_ACTIVE	0x00000004
+#define	IPMGMT_PERSIST	0x00000008
+#define	IPMGMT_INIT	0x00000010
+#define	IPMGMT_PROPS_ONLY   0x00000020
+#define	IPMGMT_UPDATE_IF    0x00000040
+#define	IPMGMT_UPDATE_IPMP  0x00000080
+
+
 /* door call command type */
 typedef enum {
 	IPMGMT_CMD_SETPROP = 1,		/* persist property */
@@ -125,7 +161,8 @@ typedef enum {
 	IPMGMT_CMD_ADDROBJ_SETLIFNUM,	/* set lifnum on the addrobj */
 	IPMGMT_CMD_ADDROBJ_ADD,		/* add addr. object to addrobj map */
 	IPMGMT_CMD_LIF2ADDROBJ,		/* lifname to addrobj mapping */
-	IPMGMT_CMD_AOBJNAME2ADDROBJ	/* aobjname to addrobj mapping */
+	IPMGMT_CMD_AOBJNAME2ADDROBJ,	/* aobjname to addrobj mapping */
+	IPMGMT_CMD_IPMP_UPDATE  /* update IPMP group members */
 } ipmgmt_door_cmd_type_t;
 
 /*
@@ -147,13 +184,6 @@ typedef struct ipmgmt_prop_arg_s {
 	char			ia_pname[MAXPROPNAMELEN];
 	char			ia_pval[MAXPROPVALLEN];
 } ipmgmt_prop_arg_t;
-/*
- * ia_flags used in ipmgmt_prop_arg_t.
- *	- APPEND updates the multi-valued property entry with a new value
- *	- REDUCE updates the multi-valued property entry by removing a value
- */
-#define	IPMGMT_APPEND	0x00000001
-#define	IPMGMT_REMOVE	0x00000002
 
 /*
  * ipadm_addr_type_t-specific values that are cached in ipmgmtd and can
@@ -182,6 +212,7 @@ typedef struct ipmgmt_if_arg_s {
 	uint32_t		ia_flags;
 	char			ia_ifname[LIFNAMSIZ];
 	sa_family_t		ia_family;
+	ipadm_if_class_t	ia_ifclass;
 } ipmgmt_if_arg_t;
 
 /* IPMGMT_CMD_INITIF door_call argument structure */
@@ -193,6 +224,14 @@ typedef struct ipmgmt_initif_arg_s {
 	/* packed nvl follows */
 } ipmgmt_initif_arg_t;
 
+/* IPMGMT_CMD_IPMP_UPDATE door_call argument structure */
+typedef struct ipmgmt_ipmp_update_arg_s {
+	ipmgmt_door_cmd_type_t	ia_cmd;
+	uint32_t	ia_flags;
+	char		ia_gifname[LIFNAMSIZ]; /* group interface name */
+	char	ia_mifname[LIFNAMSIZ]; /* group's member interface name */
+} ipmgmt_ipmp_update_arg_t;
+
 /* IPMGMT_CMD_SETADDR door_call argument */
 typedef struct ipmgmt_setaddr_arg_s {
 	ipmgmt_door_cmd_type_t	ia_cmd;
@@ -232,22 +271,6 @@ typedef struct ipmgmt_aobjop_arg_s {
 	ipadm_addr_type_t	ia_atype;
 } ipmgmt_aobjop_arg_t;
 
-/*
- * ia_flags used inside the arguments for interface/address commands
- *	- ACTIVE updates the running configuration
- *	- PERSIST updates the permanent data store
- *	- INIT	indicates that operation being performed is under init
- *		    context
- *	- PROPS_ONLY indicates the update changes the running configuration of
- *		    "props" data on the interface/address object. The props are
- *		    cached there on the parent, so a PROPS_ONLY change does not
- *		    affect the ACTIVE/PERSIST state of the parent.
- */
-#define	IPMGMT_ACTIVE		0x00000001
-#define	IPMGMT_PERSIST		0x00000002
-#define	IPMGMT_INIT		0x00000004
-#define	IPMGMT_PROPS_ONLY		0x00000008
-
 /* door call return value */
 typedef struct ipmgmt_retval_s {
 	int32_t	ir_err;
diff --git a/usr/src/lib/libipadm/common/ipadm_persist.c b/usr/src/lib/libipadm/common/ipadm_persist.c
index 3043d5e51e..557e29fb3e 100644
--- a/usr/src/lib/libipadm/common/ipadm_persist.c
+++ b/usr/src/lib/libipadm/common/ipadm_persist.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 /*
@@ -51,6 +52,12 @@
  *			wait:	DATA_TYPE_INT32
  *			primary: DATA_TYPE_BOOLEAN
  *
+ *  IPADM_NVP_FAMILIES - value holds interface families and when converted
+ *	       to nvlist, will be a DATA_TYPE_UINT16_ARRAY
+ *
+ *  IPADM_NVP_MIFNAMES - value holds IPMP group members and when converted
+ *	       to nvlist, will be a DATA_TYPE_STRING_ARRAY
+ *
  *  default  - value is a single entity and when converted to nvlist, will
  *	       contain nvpair of type DATA_TYPE_STRING. nvpairs private to
  *	       ipadm are of this type. Further the property name and property
@@ -88,21 +95,23 @@ static int ipadm_process_db_line(db_wfunc_t *, void *, FILE *fp, FILE *nfp,
 /*
  * convert nvpair to a "name=value" string for writing to the DB.
  */
-typedef size_t  ipadm_wfunc_t(nvpair_t *, char *, size_t);
+typedef size_t ipadm_wfunc_t(nvpair_t *, char *, size_t);
 
 /*
  * ipadm_rfunc_t takes (`name', `value') and adds the appropriately typed
  * nvpair to the nvlist.
  */
-typedef void  ipadm_rfunc_t(nvlist_t *, char *name, char *value);
+typedef ipadm_status_t ipadm_rfunc_t(nvlist_t *, char *, char *);
 
 static ipadm_rfunc_t	i_ipadm_str_dbline2nvl, i_ipadm_ip4_dbline2nvl,
 			i_ipadm_ip6_dbline2nvl, i_ipadm_intfid_dbline2nvl,
-			i_ipadm_dhcp_dbline2nvl;
+			i_ipadm_dhcp_dbline2nvl, i_ipadm_families_dbline2nvl,
+			i_ipadm_groupmembers_dbline2nvl;
 
 static ipadm_wfunc_t	i_ipadm_str_nvp2dbline, i_ipadm_ip4_nvp2dbline,
 			i_ipadm_ip6_nvp2dbline, i_ipadm_intfid_nvp2dbline,
-			i_ipadm_dhcp_nvp2dbline;
+			i_ipadm_dhcp_nvp2dbline, i_ipadm_families_nvp2dbline,
+			i_ipadm_groupmembers_nvp2dbline;
 
 /*
  * table of function pointers to read/write formatted entries from/to
@@ -120,6 +129,10 @@ static ipadm_conf_ent_t ipadm_conf_ent[] = {
 	{ IPADM_NVP_INTFID, i_ipadm_intfid_nvp2dbline,
 	    i_ipadm_intfid_dbline2nvl },
 	{ IPADM_NVP_DHCP, i_ipadm_dhcp_nvp2dbline, i_ipadm_dhcp_dbline2nvl },
+	{ IPADM_NVP_FAMILIES, i_ipadm_families_nvp2dbline,
+	    i_ipadm_families_dbline2nvl },
+	{ IPADM_NVP_MIFNAMES, i_ipadm_groupmembers_nvp2dbline,
+	    i_ipadm_groupmembers_dbline2nvl},
 	{ NULL,	i_ipadm_str_nvp2dbline,	i_ipadm_str_dbline2nvl }
 };
 
@@ -259,7 +272,7 @@ static size_t
 i_ipadm_dhcp_nvp2dbline(nvpair_t *nvp, char *buf, size_t buflen)
 {
 	char		addrbuf[IPADM_STRSIZE];
-	int32_t 	wait;
+	int32_t		wait;
 	boolean_t	primary;
 	nvlist_t	*v;
 
@@ -328,13 +341,13 @@ ipadm_nvlist2str(nvlist_t *nvl, char *buf, size_t buflen)
  * Adds a nvpair, using the `name' and `value', to the nvlist in `nvl'.
  * The value will be interpreted as explained at the top of this file.
  */
-static void
+static ipadm_status_t
 i_ipadm_add_nvpair(nvlist_t *nvl, char *name, char *value)
 {
 	ipadm_conf_ent_t	*ipent;
 
 	ipent = i_ipadm_find_conf_type(name);
-	(*ipent->ipent_rfunc)(nvl, name, value);
+	return ((*ipent->ipent_rfunc)(nvl, name, value));
 }
 
 /*
@@ -467,14 +480,18 @@ i_ipadm_add_dhcp2nvl(nvlist_t *nvl, boolean_t primary, int32_t wait)
 /*
  * Add (name, value) as an nvpair of type DATA_TYPE_STRING to nvlist.
  */
-static void
+static ipadm_status_t
 i_ipadm_str_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 {
+	int err;
+
 	/* if value is NULL create an empty node */
 	if (value == NULL)
-		(void) nvlist_add_string(nvl, name, "");
+		err = nvlist_add_string(nvl, name, "");
 	else
-		(void) nvlist_add_string(nvl, name, value);
+		err = nvlist_add_string(nvl, name, value);
+
+	return (ipadm_errno2status(err));
 }
 
 /*
@@ -484,7 +501,7 @@ i_ipadm_str_dbline2nvl(nvlist_t *nvl, char *name, char *value)
  * This function will add an nvlist with the hostname information in
  * nvpairs to the nvlist in `nvl'.
  */
-static void
+static ipadm_status_t
 i_ipadm_ip4_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 {
 	char			*cp, *hname;
@@ -507,7 +524,7 @@ i_ipadm_ip4_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 		(void) strlcpy(ipaddr.ipadm_static_dname, cp,
 		    sizeof (ipaddr.ipadm_static_dname));
 	}
-	(void) i_ipadm_add_ipaddr2nvl(nvl, &ipaddr);
+	return (i_ipadm_add_ipaddr2nvl(nvl, &ipaddr));
 }
 
 /*
@@ -517,7 +534,7 @@ i_ipadm_ip4_dbline2nvl(nvlist_t *nvl, char *name, char *value)
  * This function will add an nvlist with the hostname information in
  * nvpairs to the nvlist in `nvl'.
  */
-static void
+static ipadm_status_t
 i_ipadm_ip6_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 {
 	char			*cp, *hname;
@@ -540,7 +557,7 @@ i_ipadm_ip6_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 		(void) strlcpy(ipaddr.ipadm_static_dname, cp,
 		    sizeof (ipaddr.ipadm_static_dname));
 	}
-	(void) i_ipadm_add_ipaddr2nvl(nvl, &ipaddr);
+	return (i_ipadm_add_ipaddr2nvl(nvl, &ipaddr));
 }
 
 /*
@@ -548,7 +565,7 @@ i_ipadm_ip6_dbline2nvl(nvlist_t *nvl, char *name, char *value)
  * This function will add an nvlist with the address object information in
  * nvpairs to the nvlist in `nvl'.
  */
-static void
+static ipadm_status_t
 i_ipadm_intfid_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 {
 	char			*cp;
@@ -577,7 +594,7 @@ i_ipadm_intfid_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 	errno = 0;
 	ipaddr.ipadm_intfidlen = (uint32_t)strtoul(prefixlen, &endp, 10);
 	if (*endp != '\0' || errno != 0)
-		return;
+		return (ipadm_errno2status(errno));
 
 	stateless = cp;
 	stateful = strchr(stateless, ',');
@@ -587,7 +604,7 @@ i_ipadm_intfid_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 	ipaddr.ipadm_stateful = (strcmp(stateful, "yes") == 0);
 
 	/* Add all of it to the given nvlist */
-	(void) i_ipadm_add_intfid2nvl(nvl, &ipaddr);
+	return (i_ipadm_add_intfid2nvl(nvl, &ipaddr));
 }
 
 /*
@@ -595,7 +612,7 @@ i_ipadm_intfid_dbline2nvl(nvlist_t *nvl, char *name, char *value)
  * This function will add an nvlist with the dhcp address object information in
  * nvpairs to the nvlist in `nvl'.
  */
-static void
+static ipadm_status_t
 i_ipadm_dhcp_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 {
 	char		*cp;
@@ -610,39 +627,194 @@ i_ipadm_dhcp_dbline2nvl(nvlist_t *nvl, char *name, char *value)
 	errno = 0;
 	wait_time = strtol(value, &endp, 10);
 	if (*endp != '\0' || errno != 0)
-		return;
+		return (ipadm_errno2status(errno));
 	primary = (strcmp(cp, "yes") == 0);
-	(void) i_ipadm_add_dhcp2nvl(nvl, primary, (int32_t)wait_time);
+	return (i_ipadm_add_dhcp2nvl(nvl, primary, (int32_t)wait_time));
+}
+
+/*
+ * Input 'nvp': name = IPADM_NVP_FAMILIES and value = array of 'uint16_t'
+ *
+ *
+ */
+static size_t
+i_ipadm_families_nvp2dbline(nvpair_t *nvp, char *buf, size_t buflen)
+{
+	uint_t nelem = 0;
+	uint16_t *elem;
+
+	assert(nvpair_type(nvp) == DATA_TYPE_UINT16_ARRAY);
+
+	if (nvpair_value_uint16_array(nvp,
+	    &elem, &nelem) != 0) {
+		buf[0] = '\0';
+		return (0);
+	}
+
+	assert(nelem != 0 || nelem > 2);
+
+	if (nelem == 1) {
+		return (snprintf(buf, buflen, "%s=%d",
+		    nvpair_name(nvp), elem[0]));
+	} else {
+		return (snprintf(buf, buflen, "%s=%d,%d",
+		    nvpair_name(nvp), elem[0], elem[1]));
+	}
+}
+
+/*
+ * name = IPADM_NVP_FAMILIES and value = <FAMILY>[,FAMILY]
+ *
+ * output nvp: name = IPADM_NVP_FAMILIES and value = array of 'uint16_t'
+ *
+ */
+static ipadm_status_t
+i_ipadm_families_dbline2nvl(nvlist_t *nvl, char *name, char *value)
+{
+	uint16_t	families[2];
+	uint_t	nelem = 0;
+	char	*val, *lasts;
+
+	if ((val = strtok_r(value,
+	    ",", &lasts)) != NULL) {
+		families[0] = atoi(val);
+		nelem++;
+		if ((val = strtok_r(NULL,
+		    ",", &lasts)) != NULL) {
+			families[1] = atoi(val);
+			nelem++;
+		}
+		return (ipadm_errno2status(nvlist_add_uint16_array(nvl,
+		    IPADM_NVP_FAMILIES, families, nelem)));
+	}
+
+	return (IPADM_INVALID_ARG);
+}
+
+/*
+ * input nvp: name = IPADM_NVP_MIFNAMES and value = array of 'char *'
+ *
+ *
+ */
+static size_t
+i_ipadm_groupmembers_nvp2dbline(nvpair_t *nvp, char *buf, size_t buflen)
+{
+	uint_t nelem = 0;
+	char **elem;
+	size_t n;
+
+	assert(nvpair_type(nvp) == DATA_TYPE_STRING_ARRAY);
+
+	if (nvpair_value_string_array(nvp,
+	    &elem, &nelem) != 0) {
+		buf[0] = '\0';
+		return (0);
+	}
+
+	assert(nelem != 0);
+
+	n = snprintf(buf, buflen, "%s=", IPADM_NVP_MIFNAMES);
+	if (n >= buflen)
+		return (n);
+
+	while (nelem-- > 0) {
+		n = strlcat(buf, elem[nelem], buflen);
+		if (nelem > 0)
+			n = strlcat(buf, ",", buflen);
+
+		if (n > buflen)
+			return (n);
+	}
+
+	return (n);
+}
+
+/*
+ * name = IPADM_NVP_MIFNAMES and value = <if_name>[,if_name]
+ *
+ * output nvp: name = IPADM_NVP_MIFNAMES and value = array of 'char *'
+ */
+static ipadm_status_t
+i_ipadm_groupmembers_dbline2nvl(nvlist_t *nvl, char *name, char *value)
+{
+	char	**members = NULL;
+	char	*member = NULL;
+	char	*val, *lasts;
+	uint_t	m_cnt = 0;
+	ipadm_status_t	ret = IPADM_SUCCESS;
+
+	assert(strcmp(name, IPADM_NVP_MIFNAMES) == 0 && value != NULL);
+
+	for (val = strtok_r(value, ",", &lasts);
+	    val != NULL;
+	    val = strtok_r(NULL, ",", &lasts)) {
+		if ((m_cnt % 4) == 0) {
+			char **tmp = recallocarray(members, m_cnt, m_cnt + 4,
+			    sizeof (char *));
+
+			if (tmp == NULL) {
+				ret = IPADM_NO_MEMORY;
+				goto fail;
+			}
+
+			members = tmp;
+		}
+
+		member = calloc(1, LIFNAMSIZ);
+
+		if (member == NULL) {
+			ret = IPADM_NO_MEMORY;
+			goto fail;
+		}
+
+		(void) strlcpy(member, val, LIFNAMSIZ);
+		members[m_cnt++] = member;
+
+	}
+
+	if ((ret = ipadm_errno2status(nvlist_add_string_array(nvl,
+	    IPADM_NVP_MIFNAMES, members, m_cnt))) != IPADM_SUCCESS)
+		goto fail;
+
+fail:
+	while (m_cnt-- > 0) {
+		free(members[m_cnt]);
+	}
+
+	free(members);
+
+	return (ret);
 }
 
 /*
  * Parses the buffer, for name-value pairs and creates nvlist. The value
  * is always considered to be a string.
  */
-int
+ipadm_status_t
 ipadm_str2nvlist(const char *inbuf, nvlist_t **ipnvl, uint_t flags)
 {
+	ipadm_status_t	status;
 	char	*nv, *name, *val, *buf, *cp, *sep;
 	int	err;
 
 	if (inbuf == NULL || inbuf[0] == '\0' || ipnvl == NULL)
-		return (EINVAL);
+		return (IPADM_INVALID_ARG);
 	*ipnvl = NULL;
 
 	/*
 	 * If IPADM_NORVAL is set, then `inbuf' should be comma delimited values
 	 */
 	if ((flags & IPADM_NORVAL) && strchr(inbuf, '=') != NULL)
-		return (EINVAL);
+		return (IPADM_INVALID_ARG);
 
 	if ((cp = buf = strdup(inbuf)) == NULL)
-		return (errno);
+		return (ipadm_errno2status(errno));
 
 	while (isspace(*buf))
 		buf++;
 
 	if (*buf == '\0') {
-		err = EINVAL;
+		status = IPADM_INVALID_ARG;
 		goto fail;
 	}
 
@@ -658,22 +830,26 @@ ipadm_str2nvlist(const char *inbuf, nvlist_t **ipnvl, uint_t flags)
 		if ((val = strchr(nv, '=')) != NULL)
 			*val++ = '\0';
 		if (*ipnvl == NULL &&
-		    (err = nvlist_alloc(ipnvl, NV_UNIQUE_NAME, 0)) != 0)
+		    (err = nvlist_alloc(ipnvl, NV_UNIQUE_NAME, 0)) != 0) {
+			status = ipadm_errno2status(err);
 			goto fail;
+		}
 		if (nvlist_exists(*ipnvl, name)) {
-			err = EEXIST;
+			status = IPADM_EXISTS;
 			goto fail;
 		}
 		/* Add the extracted nvpair to the nvlist `ipnvl'. */
-		(void) i_ipadm_add_nvpair(*ipnvl, name, val);
+		status = i_ipadm_add_nvpair(*ipnvl, name, val);
+		if (status != IPADM_SUCCESS)
+			goto fail;
 	}
 	free(cp);
-	return (0);
+	return (IPADM_SUCCESS);
 fail:
 	free(cp);
 	nvlist_free(*ipnvl);
 	*ipnvl = NULL;
-	return (err);
+	return (status);
 }
 
 /*
diff --git a/usr/src/lib/libipadm/common/ipadm_prop.c b/usr/src/lib/libipadm/common/ipadm_prop.c
index 4fc0dc0851..889e5e2ca2 100644
--- a/usr/src/lib/libipadm/common/ipadm_prop.c
+++ b/usr/src/lib/libipadm/common/ipadm_prop.c
@@ -18,9 +18,11 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 /*
@@ -145,6 +147,11 @@ static ipadm_prop_desc_t ipadm_ip_prop_table[] = {
 	    i_ipadm_set_hostmodel, i_ipadm_get_hostmodel,
 	    i_ipadm_get_hostmodel },
 
+	{ "standby", NULL, IPADMPROP_CLASS_IF, MOD_PROTO_IP, 0,
+	    i_ipadm_set_ifprop_flags, i_ipadm_get_onoff,
+	    i_ipadm_get_ifprop_flags },
+
+
 	{ NULL, NULL, 0, 0, 0, NULL, NULL, NULL }
 };
 
@@ -598,7 +605,8 @@ i_ipadm_set_ifprop_flags(ipadm_handle_t iph, const void *arg,
 		    strcmp(pdp->ipd_name, "arp") == 0 ||
 		    strcmp(pdp->ipd_name, "nud") == 0) {
 			pval = IPADM_ONSTR;
-		} else if (strcmp(pdp->ipd_name, "forwarding") == 0) {
+		} else if (strcmp(pdp->ipd_name, "forwarding") == 0 ||
+		    strcmp(pdp->ipd_name, "standby") == 0) {
 			pval = IPADM_OFFSTR;
 		} else {
 			return (IPADM_PROP_UNKNOWN);
@@ -632,6 +640,11 @@ i_ipadm_set_ifprop_flags(ipadm_handle_t iph, const void *arg,
 			on_flags = IFF_ROUTER;
 		else
 			off_flags = IFF_ROUTER;
+	} else if (strcmp(pdp->ipd_name, "standby") == 0) {
+		if (on)
+			on_flags = IFF_STANDBY;
+		else
+			off_flags = IFF_STANDBY;
 	}
 
 	if (on_flags || off_flags)  {
@@ -649,7 +662,6 @@ i_ipadm_set_eprivport(ipadm_handle_t iph, const void *arg,
 	nvlist_t	*portsnvl = NULL;
 	nvpair_t	*nvp;
 	ipadm_status_t	status = IPADM_SUCCESS;
-	int		err;
 	uint_t		count = 0;
 
 	if (flags & IPADM_OPT_DEFAULT) {
@@ -657,8 +669,8 @@ i_ipadm_set_eprivport(ipadm_handle_t iph, const void *arg,
 		return (i_ipadm_set_prop(iph, arg, pdp, pval, proto, flags));
 	}
 
-	if ((err = ipadm_str2nvlist(pval, &portsnvl, IPADM_NORVAL)) != 0)
-		return (ipadm_errno2status(err));
+	if ((status = ipadm_str2nvlist(pval, &portsnvl, IPADM_NORVAL)) != 0)
+		return (status);
 
 	/* count the number of ports */
 	for (nvp = nvlist_next_nvpair(portsnvl, NULL); nvp != NULL;
@@ -941,7 +953,7 @@ i_ipadm_get_usesrc(ipadm_handle_t iph, const void *arg,
 	struct lifreq	lifr;
 	const char	*ifname = arg;
 	int		s;
-	char 		if_name[IF_NAMESIZE];
+	char		if_name[IF_NAMESIZE];
 	size_t		nbytes;
 
 	switch (valtype) {
@@ -984,8 +996,8 @@ i_ipadm_get_ifprop_flags(ipadm_handle_t iph, const void *arg,
     ipadm_prop_desc_t *pdp, char *buf, uint_t *bufsize, uint_t proto,
     uint_t valtype)
 {
-	uint64_t 	intf_flags;
-	char 		*val;
+	uint64_t	intf_flags;
+	char		*val;
 	size_t		nbytes;
 	const char	*ifname = arg;
 	sa_family_t	af;
@@ -1000,7 +1012,8 @@ i_ipadm_get_ifprop_flags(ipadm_handle_t iph, const void *arg,
 		    strcmp(pdp->ipd_name, "arp") == 0 ||
 		    strcmp(pdp->ipd_name, "nud") == 0) {
 			val = IPADM_ONSTR;
-		} else if (strcmp(pdp->ipd_name, "forwarding") == 0) {
+		} else if (strcmp(pdp->ipd_name, "forwarding") == 0 ||
+		    strcmp(pdp->ipd_name, "standby") == 0) {
 			val = IPADM_OFFSTR;
 		} else {
 			return (IPADM_PROP_UNKNOWN);
@@ -1026,6 +1039,9 @@ i_ipadm_get_ifprop_flags(ipadm_handle_t iph, const void *arg,
 		} else if (strcmp(pdp->ipd_name, "nud") == 0) {
 			if (!(intf_flags & IFF_NONUD))
 				val = IPADM_ONSTR;
+		} else if (strcmp(pdp->ipd_name, "standby") == 0) {
+			if (intf_flags & IFF_STANDBY)
+				val = IPADM_ONSTR;
 		}
 		nbytes = snprintf(buf, *bufsize, "%s", val);
 		break;
@@ -1060,7 +1076,7 @@ i_ipadm_get_prop(ipadm_handle_t iph, const void *arg,
 	ipadm_status_t	status = IPADM_SUCCESS;
 	const char	*ifname = arg;
 	mod_ioc_prop_t	*mip;
-	char 		*pname = pdp->ipd_name;
+	char		*pname = pdp->ipd_name;
 	uint_t		iocsize;
 
 	/* allocate sufficient ioctl buffer to retrieve value */
@@ -1154,7 +1170,7 @@ i_ipadm_populate_proparg(ipmgmt_prop_arg_t *pargp, ipadm_prop_desc_t *pdp,
  * for a given protocol `proto'. The property name is in `pname'.
  *
  * `valtype' determines the type of value that will be retrieved.
- * 	IPADM_OPT_ACTIVE -	current value of the property (active config)
+ *	IPADM_OPT_ACTIVE -	current value of the property (active config)
  *	IPADM_OPT_PERSIST -	value of the property from persistent store
  *	IPADM_OPT_DEFAULT -	default hard coded value (boot-time value)
  *	IPADM_OPT_PERM -	read/write permissions for the value
@@ -1302,9 +1318,9 @@ i_ipadm_set_prop(ipadm_handle_t iph, const void *arg,
 {
 	ipadm_status_t	status = IPADM_SUCCESS;
 	const char	*ifname = arg;
-	mod_ioc_prop_t 	*mip;
-	char 		*pname = pdp->ipd_name;
-	uint_t 		valsize, iocsize;
+	mod_ioc_prop_t	*mip;
+	char		*pname = pdp->ipd_name;
+	uint_t		valsize, iocsize;
 	uint_t		iocflags = 0;
 
 	if (flags & IPADM_OPT_DEFAULT) {
@@ -1365,7 +1381,7 @@ i_ipadm_setprop_common(ipadm_handle_t iph, const char *ifname,
     const char *pname, const char *buf, uint_t proto, uint_t pflags)
 {
 	ipadm_status_t		status = IPADM_SUCCESS;
-	boolean_t 		persist = (pflags & IPADM_OPT_PERSIST);
+	boolean_t		persist = (pflags & IPADM_OPT_PERSIST);
 	boolean_t		reset = (pflags & IPADM_OPT_DEFAULT);
 	ipadm_prop_desc_t	*pdp;
 	boolean_t		is_if = (ifname != NULL);
@@ -1701,8 +1717,7 @@ i_ipadm_persist_propval(ipadm_handle_t iph, ipadm_prop_desc_t *pdp,
  * This is called from ipadm_set_ifprop() to validate the set operation.
  * It does the following steps:
  * 1. Validates the interface name.
- * 2. Fails if it is an IPMP meta-interface or an underlying interface.
- * 3. In case of a persistent operation, verifies that the
+ * 2. In case of a persistent operation, verifies that the
  *	interface is persistent.
  */
 static ipadm_status_t
@@ -1719,12 +1734,6 @@ i_ipadm_validate_if(ipadm_handle_t iph, const char *ifname,
 		return (IPADM_INVALID_ARG);
 
 	af = (proto == MOD_PROTO_IPV6 ? AF_INET6 : AF_INET);
-	/*
-	 * Setting properties on an IPMP meta-interface or underlying
-	 * interface is not supported.
-	 */
-	if (i_ipadm_is_ipmp(iph, ifname) || i_ipadm_is_under_ipmp(iph, ifname))
-		return (IPADM_NOTSUP);
 
 	/* Check if interface exists in the persistent configuration. */
 	status = i_ipadm_if_pexists(iph, ifname, af, &p_exists);
diff --git a/usr/src/lib/libipadm/common/libipadm.c b/usr/src/lib/libipadm/common/libipadm.c
index 9ef32f35c1..005d86fc0c 100644
--- a/usr/src/lib/libipadm/common/libipadm.c
+++ b/usr/src/lib/libipadm/common/libipadm.c
@@ -22,8 +22,8 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2015 Joyent, Inc.
- * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 #include <stdio.h>
@@ -537,39 +537,6 @@ i_ipadm_is_6to4(ipadm_handle_t iph, char *ifname)
 }
 
 /*
- * Returns B_TRUE if `ifname' represents an IPMP underlying interface.
- */
-boolean_t
-i_ipadm_is_under_ipmp(ipadm_handle_t iph, const char *ifname)
-{
-	struct lifreq	lifr;
-
-	(void) strlcpy(lifr.lifr_name, ifname, sizeof (lifr.lifr_name));
-	if (ioctl(iph->iph_sock, SIOCGLIFGROUPNAME, (caddr_t)&lifr) < 0) {
-		if (ioctl(iph->iph_sock6, SIOCGLIFGROUPNAME,
-		    (caddr_t)&lifr) < 0) {
-			return (B_FALSE);
-		}
-	}
-	return (lifr.lifr_groupname[0] != '\0');
-}
-
-/*
- * Returns B_TRUE if `ifname' represents an IPMP meta-interface.
- */
-boolean_t
-i_ipadm_is_ipmp(ipadm_handle_t iph, const char *ifname)
-{
-	uint64_t flags;
-
-	if (i_ipadm_get_flags(iph, ifname, AF_INET, &flags) != IPADM_SUCCESS &&
-	    i_ipadm_get_flags(iph, ifname, AF_INET6, &flags) != IPADM_SUCCESS)
-		return (B_FALSE);
-
-	return ((flags & IFF_IPMP) != 0);
-}
-
-/*
  * For a given interface name, ipadm_if_enabled() checks if v4
  * or v6 or both IP interfaces exist in the active configuration.
  */
@@ -702,40 +669,82 @@ i_ipadm_init_ifobj(ipadm_handle_t iph, const char *ifname, nvlist_t *ifnvl)
 {
 	nvlist_t	*nvl = NULL;
 	nvpair_t	*nvp;
-	char		*afstr;
-	ipadm_status_t	status;
+	ipadm_status_t	status = IPADM_ENXIO;
 	ipadm_status_t	ret_status = IPADM_SUCCESS;
 	char		newifname[LIFNAMSIZ];
 	char		*aobjstr;
-	sa_family_t	af = AF_UNSPEC;
-	boolean_t	is_ngz = (iph->iph_zoneid != GLOBAL_ZONEID);
+	uint16_t	*afs;
+	char		*gifname;
+	uint_t		nelem = 0;
+	boolean_t	init_from_gz = B_FALSE;
+	boolean_t	move_to_group = B_FALSE;
 
 	(void) strlcpy(newifname, ifname, sizeof (newifname));
+
 	/*
-	 * First plumb the given interface and then apply all the persistent
-	 * interface properties and then instantiate any persistent addresses
-	 * objects on that interface.
+	 * First go through the ifnvl nvlist looking for nested nvlist
+	 * containing interface class and address families.
 	 */
 	for (nvp = nvlist_next_nvpair(ifnvl, NULL); nvp != NULL;
 	    nvp = nvlist_next_nvpair(ifnvl, nvp)) {
-		if (nvpair_value_nvlist(nvp, &nvl) != 0)
+		char *icstr;
+		char **mifnames;
+		uint32_t ipadm_flags = IPADM_OPT_ACTIVE;
+
+		if (nvpair_value_nvlist(nvp, &nvl) != 0 ||
+		    nvlist_lookup_uint16_array(nvl, IPADM_NVP_FAMILIES,
+		    &afs, &nelem) != 0)
 			continue;
 
-		if (nvlist_lookup_string(nvl, IPADM_NVP_FAMILY, &afstr) == 0) {
-			status = i_ipadm_plumb_if(iph, newifname, atoi(afstr),
-			    IPADM_OPT_ACTIVE);
-			/*
-			 * If the interface is already plumbed, we should
-			 * ignore this error because there might be address
-			 * address objects on that interface that needs to
-			 * be enabled again.
-			 */
+		/* Check if this is IPMP group interface */
+		if (nvlist_lookup_string(nvl, IPADM_NVP_IFCLASS,
+		    &icstr) == 0 && atoi(icstr) == IPADM_IF_CLASS_IPMP)
+			ipadm_flags |= IPADM_OPT_IPMP;
+
+		/* Create interfaces for address families specified */
+		while (nelem-- > 0) {
+			uint16_t af = afs[nelem];
+
+			assert(af == AF_INET || af == AF_INET6);
+
+			status = i_ipadm_plumb_if(iph, newifname, af,
+			    ipadm_flags);
 			if (status == IPADM_IF_EXISTS)
 				status = IPADM_SUCCESS;
+			if (status != IPADM_SUCCESS)
+				return (status);
+		}
+		if (nvlist_lookup_string(nvl, IPADM_NVP_GIFNAME,
+		    &gifname) == 0) {
+			/*
+			 * IPMP underlying interface. Move to the
+			 * specified IPMP group.
+			 */
+			move_to_group = B_TRUE;
+		} else if ((ipadm_flags & IPADM_OPT_IPMP) &&
+		    nvlist_lookup_string_array(nvl, IPADM_NVP_MIFNAMES,
+		    &mifnames, &nelem) == 0) {
+			/* Non-empty IPMP group interface */
+			while (nelem-- > 0) {
+				(void) ipadm_add_ipmp_member(iph, newifname,
+				    mifnames[nelem], IPADM_OPT_ACTIVE);
+			}
+		}
+		if (iph->iph_zoneid != GLOBAL_ZONEID)
+			init_from_gz = B_TRUE;
+	}
 
-			if (is_ngz)
-				af = atoi(afstr);
-		} else if (nvlist_lookup_string(nvl, IPADM_NVP_AOBJNAME,
+	if (status != IPADM_SUCCESS)
+		return (status);
+
+	/*
+	 * Go through the ifnvl nvlist again, applying persistent configuration.
+	 */
+	for (nvp = nvlist_next_nvpair(ifnvl, NULL); nvp != NULL;
+	    nvp = nvlist_next_nvpair(ifnvl, nvp)) {
+		if (nvpair_value_nvlist(nvp, &nvl) != 0)
+			continue;
+		if (nvlist_lookup_string(nvl, IPADM_NVP_AOBJNAME,
 		    &aobjstr) == 0) {
 			/*
 			 * For addresses, we need to relocate addrprops from the
@@ -746,10 +755,12 @@ i_ipadm_init_ifobj(ipadm_handle_t iph, const char *ifname, nvlist_t *ifnvl)
 			    nvlist_exists(nvl, IPADM_NVP_DHCP)) {
 				status = i_ipadm_merge_addrprops_from_nvl(ifnvl,
 				    nvl, aobjstr);
+
 				if (status != IPADM_SUCCESS)
 					continue;
 			}
 			status = i_ipadm_init_addrobj(iph, nvl);
+
 			/*
 			 * If this address is in use on some other interface,
 			 * we want to record an error to be returned as
@@ -760,15 +771,17 @@ i_ipadm_init_ifobj(ipadm_handle_t iph, const char *ifname, nvlist_t *ifnvl)
 				ret_status = IPADM_ALL_ADDRS_NOT_ENABLED;
 				status = IPADM_SUCCESS;
 			}
-		} else {
-			assert(nvlist_exists(nvl, IPADM_NVP_PROTONAME));
+		} else if (nvlist_exists(nvl, IPADM_NVP_PROTONAME) == B_TRUE) {
 			status = i_ipadm_init_ifprop(iph, nvl);
 		}
 		if (status != IPADM_SUCCESS)
 			return (status);
 	}
-
-	if (is_ngz && af != AF_UNSPEC)
+	if (move_to_group) {
+		(void) ipadm_add_ipmp_member(iph, gifname, newifname,
+		    IPADM_OPT_ACTIVE);
+	}
+	if (init_from_gz)
 		ret_status = ipadm_init_net_from_gz(iph, newifname, NULL);
 	return (ret_status);
 }
@@ -788,8 +801,9 @@ i_ipadm_init_ifs(ipadm_handle_t iph, const char *ifs, nvlist_t **allifs)
 	int			err;
 	ipadm_status_t		status = IPADM_SUCCESS;
 
-	if ((err = ipadm_str2nvlist(ifs, &nvl, IPADM_NORVAL)) != 0)
-		return (ipadm_errno2status(err));
+	status = ipadm_str2nvlist(ifs, &nvl, IPADM_NORVAL);
+	if (status != IPADM_SUCCESS)
+		return (status);
 
 	err = nvlist_pack(nvl, &nvlbuf, &nvlsize, NV_ENCODE_NATIVE, 0);
 	if (err != 0) {
@@ -981,6 +995,36 @@ reopen:
 }
 
 /*
+ * A helper that is used by i_ipadm_get_db_addr and i_ipadm_get_db_if
+ * to do a door_call to ipmgmtd, that should return persistent information
+ * about interfaces or/and addresses from ipadm DB
+ */
+ipadm_status_t
+i_ipadm_call_ipmgmtd(ipadm_handle_t iph, void *garg, size_t garg_size,
+    nvlist_t **onvl)
+{
+	ipmgmt_get_rval_t	*rvalp;
+	int			err;
+	size_t			nvlsize;
+	char			*nvlbuf;
+
+	rvalp = malloc(sizeof (ipmgmt_get_rval_t));
+	if (rvalp == NULL)
+		return (IPADM_NO_MEMORY);
+
+	err = ipadm_door_call(iph, garg, garg_size, (void **)&rvalp,
+	    sizeof (*rvalp), B_TRUE);
+	if (err == 0) {
+		nvlsize = rvalp->ir_nvlsize;
+		nvlbuf = (char *)rvalp + sizeof (ipmgmt_get_rval_t);
+		err = nvlist_unpack(nvlbuf, nvlsize, onvl, 0);
+	}
+	free(rvalp);
+
+	return (ipadm_errno2status(err));
+}
+
+/*
  * ipadm_is_nil_hostname() : Determine if the `hostname' is nil: i.e.,
  *			NULL, empty, or a single space (e.g., as returned by
  *			domainname(1M)/sysinfo).
diff --git a/usr/src/lib/libipadm/common/libipadm.h b/usr/src/lib/libipadm/common/libipadm.h
index 0ae9d89e4b..66e7888d2b 100644
--- a/usr/src/lib/libipadm/common/libipadm.h
+++ b/usr/src/lib/libipadm/common/libipadm.h
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
- * Copyright 2021, Tintri by DDN. All rights reserved.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 #ifndef _LIBIPADM_H
 #define	_LIBIPADM_H
@@ -39,6 +39,8 @@ extern "C" {
 #include <libnvpair.h>
 #include <netinet/tcp.h>
 #include <sys/stropts.h>
+#include <sys/list.h>
+#include <stddef.h>
 
 #define	IPADM_AOBJ_USTRSIZ	32
 #define	IPADM_AOBJSIZ		(LIFNAMSIZ + IPADM_AOBJ_USTRSIZ)
@@ -206,27 +208,31 @@ typedef enum {
 	IFIS_DISABLED		/* Interface has been disabled. */
 } ipadm_if_state_t;
 
-/*
- * Declare ipadm_if_info_list_t as a container for ipadm_if_info_t.
- *
- * ipadm_if_info_t used to have a list pointer ifi_next for linking a number
- * of ipadm_if_info_t's together. Even though this linking wasn't used in the
- * data exchange between ipmgmtd and libipadm, this meant the structure wasn't
- * safe for passing through the door between 32bit and 64bit processes.
- */
+typedef list_t ipadm_ipmp_members_t;
+
+typedef struct {
+    list_node_t node;
+    char if_name[LIFNAMSIZ];
+} ipadm_ipmp_member_t;
+
+typedef enum {
+    IPADM_IF_CLASS_REGULAR,
+    IPADM_IF_CLASS_IPMP,
+    IPADM_IF_CLASS_VIRTUAL,
+    IPADM_IF_CLASS_UNKNOWN
+} ipadm_if_class_t;
+
 typedef struct ipadm_if_info_s {
+	struct ipadm_if_info_s	*ifi_next;
 	char			ifi_name[LIFNAMSIZ];	/* interface name */
+	ipadm_if_class_t	ifi_class;		/* interface class */
 	ipadm_if_state_t	ifi_state;		/* see above */
 	uint_t			ifi_cflags;		/* current flags */
 	uint_t			ifi_pflags;		/* persistent flags */
+	ipadm_ipmp_members_t    ifi_ipmp_cmembers; /* current IPMP members */
+	ipadm_ipmp_members_t    ifi_ipmp_pmembers; /* persistent IPMP members */
 } ipadm_if_info_t;
 
-typedef struct ipadm_if_info_list_s {
-	struct ipadm_if_info_list_s	*ifil_next;
-	ipadm_if_info_t			ifil_ifi;
-} ipadm_if_info_list_t;
-
-
 /* ipadm_if_info_t flags */
 #define	IFIF_BROADCAST		0x00000001
 #define	IFIF_MULTICAST		0x00000002
@@ -293,11 +299,15 @@ extern ipadm_status_t	ipadm_disable_if(ipadm_handle_t, const char *,
 			    uint32_t);
 extern ipadm_status_t	ipadm_enable_if(ipadm_handle_t, const char *, uint32_t);
 extern ipadm_status_t	ipadm_if_info(ipadm_handle_t, const char *,
-			    ipadm_if_info_list_t **, uint32_t, int64_t);
-extern void		ipadm_free_if_info(ipadm_if_info_list_t *);
+			    ipadm_if_info_t **, uint32_t, int64_t);
+extern void		ipadm_free_if_info(ipadm_if_info_t *);
 extern ipadm_status_t	ipadm_delete_if(ipadm_handle_t, const char *,
 			    sa_family_t, uint32_t);
 extern void		ipadm_if_move(ipadm_handle_t, const char *);
+extern ipadm_status_t ipadm_add_ipmp_member(ipadm_handle_t, const char *,
+	    const char *, uint32_t);
+extern ipadm_status_t ipadm_remove_ipmp_member(ipadm_handle_t, const char *,
+	    const char *, uint32_t);
 
 /*
  * Address management functions
@@ -377,7 +387,7 @@ extern ipadm_status_t	ipadm_get_prop(ipadm_handle_t, const char *, char *,
  * miscellaneous helper functions.
  */
 extern const char	*ipadm_status2str(ipadm_status_t);
-extern int		ipadm_str2nvlist(const char *, nvlist_t **, uint_t);
+extern ipadm_status_t	ipadm_str2nvlist(const char *, nvlist_t **, uint_t);
 extern size_t		ipadm_nvlist2str(nvlist_t *, char *, size_t);
 extern char		*ipadm_proto2str(uint_t);
 extern uint_t		ipadm_str2proto(const char *);
diff --git a/usr/src/lib/libipadm/common/libipadm_impl.h b/usr/src/lib/libipadm/common/libipadm_impl.h
index 98b95092d9..1766558db6 100644
--- a/usr/src/lib/libipadm/common/libipadm_impl.h
+++ b/usr/src/lib/libipadm/common/libipadm_impl.h
@@ -18,10 +18,12 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
+ * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
  */
 
 #ifndef _LIBIPADM_IMPL_H
@@ -32,6 +34,7 @@ extern "C" {
 #endif
 
 #include <sys/socket.h>
+#include <sys/list.h>
 #include <net/if.h>
 #include <libipadm.h>
 #include <libdladm.h>
@@ -52,6 +55,11 @@ extern "C" {
 /* mask for flags accepted by libipadm functions */
 #define	IPADM_COMMON_OPT_MASK	(IPADM_OPT_ACTIVE | IPADM_OPT_PERSIST)
 
+typedef enum {
+    IPADM_ADD_IPMP,
+    IPADM_REMOVE_IPMP
+} ipadm_ipmp_op_t;
+
 /* Opaque library handle */
 struct ipadm_handle {
 	int		iph_sock;	/* socket to interface */
@@ -65,7 +73,7 @@ struct ipadm_handle {
 };
 
 struct ipadm_addrobj_s {
-	char 			ipadm_ifname[LIFNAMSIZ];
+	char			ipadm_ifname[LIFNAMSIZ];
 	int32_t			ipadm_lifnum;
 	char			ipadm_aobjname[IPADM_AOBJSIZ];
 	ipadm_addr_type_t	ipadm_atype;
@@ -154,15 +162,15 @@ extern ipadm_status_t	i_ipadm_delete_addr(ipadm_handle_t, ipadm_addrobj_t);
 extern int		i_ipadm_strioctl(int, int, char *, int);
 extern boolean_t	i_ipadm_is_loopback(const char *);
 extern boolean_t	i_ipadm_is_vni(const char *);
-extern boolean_t	i_ipadm_is_ipmp(ipadm_handle_t, const char *);
-extern boolean_t	i_ipadm_is_under_ipmp(ipadm_handle_t, const char *);
 extern boolean_t	i_ipadm_is_6to4(ipadm_handle_t, char *);
 extern boolean_t	i_ipadm_validate_ifname(ipadm_handle_t, const char *);
 extern ipadm_status_t	ipadm_errno2status(int);
 extern int		ipadm_door_call(ipadm_handle_t, void *, size_t, void **,
 			    size_t, boolean_t);
-extern boolean_t 	ipadm_if_enabled(ipadm_handle_t, const char *,
+extern boolean_t	ipadm_if_enabled(ipadm_handle_t, const char *,
 			    sa_family_t);
+extern ipadm_status_t i_ipadm_call_ipmgmtd(ipadm_handle_t, void *,
+	    size_t, nvlist_t **);
 
 /* ipadm_ndpd.c */
 extern	ipadm_status_t	i_ipadm_create_ipv6addrs(ipadm_handle_t,
@@ -187,6 +195,8 @@ extern ipadm_status_t	i_ipadm_get_persist_propval(ipadm_handle_t,
 			    const void *);
 
 /* ipadm_addr.c */
+extern ipadm_status_t	i_ipadm_active_addr_info(ipadm_handle_t, const char *,
+			    ipadm_addr_info_t **, uint32_t, int64_t);
 extern void		i_ipadm_init_addr(ipadm_addrobj_t, const char *,
 			    const char *, ipadm_addr_type_t);
 extern ipadm_status_t	i_ipadm_merge_addrprops_from_nvl(nvlist_t *, nvlist_t *,
@@ -230,6 +240,12 @@ extern ipadm_status_t	i_ipadm_delete_ifobj(ipadm_handle_t, const char *,
 			    sa_family_t, boolean_t);
 extern int		i_ipadm_get_lnum(const char *);
 
+extern ipadm_status_t i_ipadm_set_groupname_active(ipadm_handle_t,
+	    const char *, const char *);
+extern ipadm_status_t i_ipadm_get_groupname_active(ipadm_handle_t,
+	    const char *, char *, size_t);
+extern boolean_t i_ipadm_is_under_ipmp(ipadm_handle_t, const char *);
+extern boolean_t i_ipadm_is_ipmp(ipadm_handle_t, const char *);
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/lib/libipadm/common/mapfile-vers b/usr/src/lib/libipadm/common/mapfile-vers
index 7b03b77f33..7e2becc2bc 100644
--- a/usr/src/lib/libipadm/common/mapfile-vers
+++ b/usr/src/lib/libipadm/common/mapfile-vers
@@ -21,6 +21,7 @@
 #
 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright (c) 2016, Chris Fraire <cfraire@me.com>.
+# Copyright 2021 Tintri by DDN, Inc. All rights reserved.
 #
 
 #
@@ -41,6 +42,7 @@ $mapfile_version 2
 
 SYMBOL_VERSION SUNWprivate_1.1 {
     global:
+	ipadm_add_ipmp_member;
 	ipadm_add_aobjname;
 	ipadm_addr_info;
 	ipadm_check_auth;
@@ -79,6 +81,7 @@ SYMBOL_VERSION SUNWprivate_1.1 {
 	ipadm_open_arp_on_udp;
 	ipadm_proto2str;
 	ipadm_refresh_addr;
+	ipadm_remove_ipmp_member;
 	ipadm_rw_db;
 	ipadm_set_addr;
 	ipadm_set_addrprop;