9894 Deliver python3 modules

9904 Split python modules out into separate packages
5571 Provide 64-bit python modules
Reviewed by: Alexander Pyhalov <apyhalov@gmail.com>
Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Dan McDonald <danmcd@joyent.com>

13 files changed, 214 insertions, 45 deletions

diff --git a/usr/src/lib/pyzfs/Makefile b/usr/src/lib/pyzfs/Makefile
index c69270f58f..2cf7b8a517 100644
--- a/usr/src/lib/pyzfs/Makefile
+++ b/usr/src/lib/pyzfs/Makefile
@@ -21,11 +21,15 @@
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 include	../Makefile.lib
 
-SUBDIRS=	$(MACH)
+SUBDIRS=
+$(BUILDPY2)SUBDIRS += $(MACH)
+$(BUILD64)$(BUILDPY2)SUBDIRS += $(MACH64)
+$(BUILD64)$(BUILDPY3)SUBDIRS += py3
 XGETTEXT=	$(GNUXGETTEXT)
 XGETFLAGS=	$(GNUXGETFLAGS)
 
@@ -38,7 +42,7 @@ lint :=		TARGET= lint
 MSGFIND =	$(FIND) . -name '*.py' -o -name '*.c'
 MSGFILES =	$(MSGFIND:sh)
 PYCFIND =	$(FIND) . -name '*.pyc'
-PYCFILES = 	$(PYCFIND:sh)
+PYCFILES =	$(PYCFIND:sh)
 POFILE =	pyzfs.po
 
 .KEEP_STATE:
diff --git a/usr/src/lib/pyzfs/Makefile.com b/usr/src/lib/pyzfs/Makefile.com
index 59720a8334..347b4316eb 100644
--- a/usr/src/lib/pyzfs/Makefile.com
+++ b/usr/src/lib/pyzfs/Makefile.com
@@ -20,6 +20,7 @@
 #
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 LIBRARY =	ioctl.a
@@ -27,15 +28,15 @@ VERS =
 OBJECTS =	ioctl.o
 
 PYSRCS=		__init__.py util.py dataset.py \
-	allow.py unallow.py \
-	userspace.py groupspace.py holds.py table.py
-
+		allow.py unallow.py \
+		userspace.py groupspace.py holds.py table.py
 
 include ../../Makefile.lib
 
-LIBLINKS = 
+LIBLINKS =
 SRCDIR =	../common
 ROOTLIBDIR=	$(ROOT)/usr/lib/python$(PYTHON_VERSION)/vendor-packages/zfs
+ROOTLIBDIR64=	$(ROOTLIBDIR)/64
 PYOBJS=		$(PYSRCS:%.py=$(SRCDIR)/%.pyc)
 PYFILES=	$(PYSRCS) $(PYSRCS:%.py=%.pyc)
 ROOTPYZFSFILES= $(PYFILES:%=$(ROOTLIBDIR)/%)
@@ -44,18 +45,17 @@ CSTD=        $(CSTD_GNU99)
 C99LMODE=       -Xc99=%all
 
 LIBS =		$(DYNLIB)
-LDLIBS +=	-lc -lnvpair -lpython$(PYTHON_VERSION) -lzfs
+LDLIBS +=	-lc -lnvpair -lpython$(PYTHON_VERSION)$(PYTHON_SUFFIX) -lzfs
 CFLAGS +=	$(CCVERBOSE)
 CERRWARN +=	-_gcc=-Wno-unused-variable
-CPPFLAGS +=	-I$(ADJUNCT_PROTO)/usr/include/python$(PYTHON_VERSION)
+CPPFLAGS +=	\
+	-I$(ADJUNCT_PROTO)/usr/include/python$(PYTHON_VERSION)$(PYTHON_SUFFIX)
 CPPFLAGS +=	-I../../../uts/common/fs/zfs
 CPPFLAGS +=	-I../../../common/zfs
 
 .KEEP_STATE:
 
-all: $(PYOBJS) $(LIBS)
-
-install: all $(ROOTPYZFSFILES)
+all:
 
 $(ROOTLIBDIR)/%: %
 	$(INS.pyfile)
@@ -63,6 +63,12 @@ $(ROOTLIBDIR)/%: %
 $(ROOTLIBDIR)/%: ../common/%
 	$(INS.pyfile)
 
+$(ROOTLIBDIR64)/%: %
+	$(INS.pyfile)
+
+$(ROOTLIBDIR64)/%: ../common/%
+	$(INS.pyfile)
+
 lint: lintcheck
 
 include ../../Makefile.targ
diff --git a/usr/src/lib/pyzfs/amd64/Makefile b/usr/src/lib/pyzfs/amd64/Makefile
new file mode 100644
index 0000000000..f3fde27d22
--- /dev/null
+++ b/usr/src/lib/pyzfs/amd64/Makefile
@@ -0,0 +1,21 @@
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+# CDDL HEADER END
+
+#
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+#
+
+include ../Makefile.com
+include ../../Makefile.lib.64
+
+all: $(LIBS)
+install: all $(ROOTLIBS64)
diff --git a/usr/src/lib/pyzfs/common/allow.py b/usr/src/lib/pyzfs/common/allow.py
index 2c01280b4b..91d4834679 100644
--- a/usr/src/lib/pyzfs/common/allow.py
+++ b/usr/src/lib/pyzfs/common/allow.py
@@ -21,6 +21,7 @@
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
 # Copyright (c) 2013 by Delphix. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """This module implements the "zfs allow" and "zfs unallow" subcommands.
@@ -146,7 +147,7 @@ def args_to_perms(parser, options, who, perms):
 		baseperms = None
 
 	d = dict()
-	
+
 	def storeperm(typechr, inheritchr, arg):
 		assert typechr in "ugecs"
 		assert inheritchr in "ld-"
@@ -251,20 +252,20 @@ def canonicalized_perm(permname):
 	except KeyError:
 		raise zfs.util.ZFSError(errno.EINVAL, permname,
 		    _("invalid permission"))
-		
+
 def print_perms():
 	"""Print the set of supported permissions."""
 	print(_("\nThe following permissions are supported:\n"))
 	fmt = "%-16s %-14s\t%s"
 	print(fmt % (_("NAME"), _("TYPE"), _("NOTES")))
 
-	for (name, note) in sorted(perms_subcmd.iteritems()):
+	for (name, note) in sorted(perms_subcmd.items()):
 		print(fmt % (name, _("subcommand"), note))
 
-	for (name, note) in sorted(perms_other.iteritems()):
+	for (name, note) in sorted(perms_other.items()):
 		print(fmt % (name, _("other"), note))
 
-	for (name, prop) in sorted(zfs.dataset.proptable.iteritems()):
+	for (name, prop) in sorted(zfs.dataset.proptable.items()):
 		if prop.visible and prop.delegatable():
 			print(fmt % (name, _("property"), ""))
 
@@ -335,7 +336,6 @@ def do_allow():
 			print(s + "-" * (70-len(s)))
 			print(p[fs])
 		return
-	
 
 	(options, args) = parser.parse_args(sys.argv[2:])
 
diff --git a/usr/src/lib/pyzfs/common/dataset.py b/usr/src/lib/pyzfs/common/dataset.py
index 9d4652235a..752449486f 100644
--- a/usr/src/lib/pyzfs/common/dataset.py
+++ b/usr/src/lib/pyzfs/common/dataset.py
@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """Implements the Dataset class, providing methods for manipulating ZFS
@@ -69,7 +70,7 @@ class Property(object):
 		return self.attr != "readonly"
 
 proptable = dict()
-for name, t in zfs.ioctl.get_proptable().iteritems():
+for name, t in zfs.ioctl.get_proptable().items():
 	proptable[name] = Property(t)
 del name, t
 
@@ -79,7 +80,7 @@ def getpropobj(name):
 	try:
 		return proptable[name]
 	except KeyError:
-		for p in proptable.itervalues():
+		for p in proptable.values():
 			if p.colname and p.colname.lower() == name:
 				return p
 		raise
@@ -90,7 +91,7 @@ class Dataset(object):
 	Generally, this class provides interfaces to the C functions in
 	zfs.ioctl which actually interface with the kernel to manipulate
 	datasets.
-	
+
 	Unless otherwise noted, any method can raise a ZFSError to
 	indicate failure."""
 
@@ -101,7 +102,7 @@ class Dataset(object):
 	    types=("filesystem", "volume"), snaps=True):
 		"""Open the named dataset, checking that it exists and
 		is of the specified type.
-		
+
 		name is the string name of this dataset.
 
 		props is the property settings dict from zfs.ioctl.next_dataset.
@@ -137,7 +138,7 @@ class Dataset(object):
 
 		Currently only works for native properties (those with a
 		Property object.)
-		
+
 		Raises KeyError if propname does not specify a native property.
 		Does not raise ZFSError.
 		"""
@@ -165,7 +166,7 @@ class Dataset(object):
 			yield ds
 			for child in ds.descendents():
 				yield child
-	
+
 	def userspace(self, prop):
 		"""A generator function which iterates over a
 		userspace-type property.
@@ -177,14 +178,14 @@ class Dataset(object):
 		"""
 
 		d = zfs.ioctl.userspace_many(self.name, prop)
-		for ((domain, rid), space) in d.iteritems():
+		for ((domain, rid), space) in d.items():
 			yield (domain, rid, space)
 
 	def userspace_upgrade(self):
 		"""Initialize the accounting information for
 		userused@... and groupused@... properties."""
 		return zfs.ioctl.userspace_upgrade(self.name)
-	
+
 	def set_fsacl(self, un, d):
 		"""Add to the "zfs allow"-ed permissions on this Dataset.
 
@@ -219,7 +220,7 @@ def snapshots_fromcmdline(dsnames, recursive):
 		try:
 			ds = Dataset(dsname)
 			yield ds
-		except zfs.util.ZFSError, e:
+		except zfs.util.ZFSError as e:
 			if not recursive or e.errno != errno.ENOENT:
 				raise
 		if recursive:
@@ -229,6 +230,6 @@ def snapshots_fromcmdline(dsnames, recursive):
 				try:
 					yield Dataset(child.name + "@" +
 					    snapname)
-				except zfs.util.ZFSError, e:
+				except zfs.util.ZFSError as e:
 					if e.errno != errno.ENOENT:
 						raise
diff --git a/usr/src/lib/pyzfs/common/holds.py b/usr/src/lib/pyzfs/common/holds.py
index 0a1508e76a..63645e329c 100644
--- a/usr/src/lib/pyzfs/common/holds.py
+++ b/usr/src/lib/pyzfs/common/holds.py
@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """This module implements the "zfs holds" subcommand.
@@ -59,12 +60,12 @@ def do_holds():
 
 	fields = ("name", "tag", "timestamp")
 	rjustfields = ()
-	printing = False 
+	printing = False
 	gotone = False
-	t = zfs.table.Table(fields, rjustfields) 
+	t = zfs.table.Table(fields, rjustfields)
 	for ds in zfs.dataset.snapshots_fromcmdline(args, options.recursive):
 		gotone = True
-		for tag, tm in ds.get_holds().iteritems():
+		for tag, tm in ds.get_holds().items():
 			val = {"name": ds.name, "tag": tag,
 			    "timestamp": time.ctime(tm)}
 			t.addline(ds.name, val)
diff --git a/usr/src/lib/pyzfs/common/ioctl.c b/usr/src/lib/pyzfs/common/ioctl.c
index d8c0d18679..55c03fc2b8 100644
--- a/usr/src/lib/pyzfs/common/ioctl.c
+++ b/usr/src/lib/pyzfs/common/ioctl.c
@@ -21,6 +21,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <Python.h>
@@ -131,7 +132,7 @@ dict2nvl(PyObject *d)
 	nvlist_t *nvl;
 	int err;
 	PyObject *key, *value;
-	int pos = 0;
+	Py_ssize_t pos = 0;
 
 	if (!PyDict_Check(d)) {
 		PyErr_SetObject(PyExc_ValueError, d);
@@ -142,7 +143,11 @@ dict2nvl(PyObject *d)
 	assert(err == 0);
 
 	while (PyDict_Next(d, &pos, &key, &value)) {
+#if PY_MAJOR_VERSION >= 3
+		char *keystr = PyBytes_AsString(key);
+#else
 		char *keystr = PyString_AsString(key);
+#endif
 		if (keystr == NULL) {
 			PyErr_SetObject(PyExc_KeyError, key);
 			nvlist_free(nvl);
@@ -155,11 +160,19 @@ dict2nvl(PyObject *d)
 			nvlist_free(valnvl);
 		} else if (value == Py_None) {
 			err = nvlist_add_boolean(nvl, keystr);
+#if PY_MAJOR_VERSION >= 3
+		} else if (PyBytes_Check(value)) {
+#else
 		} else if (PyString_Check(value)) {
+#endif
+#if PY_MAJOR_VERSION >= 3
+			char *valstr = PyBytes_AsString(value);
+#else
 			char *valstr = PyString_AsString(value);
+#endif
 			err = nvlist_add_string(nvl, keystr, valstr);
-		} else if (PyInt_Check(value)) {
-			uint64_t valint = PyInt_AsUnsignedLongLongMask(value);
+		} else if (PyLong_Check(value)) {
+			uint64_t valint = PyLong_AsUnsignedLongLongMask(value);
 			err = nvlist_add_uint64(nvl, keystr, valint);
 		} else if (PyBool_Check(value)) {
 			boolean_t valbool = value == Py_True ? B_TRUE : B_FALSE;
@@ -525,19 +538,49 @@ static PyMethodDef zfsmethods[] = {
 	{NULL, NULL, 0, NULL}
 };
 
-void
-initioctl(void)
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef zfs_module = {
+	PyModuleDef_HEAD_INIT,
+	"zfs.ioctl",
+	NULL,
+	-1,
+	zfsmethods
+};
+#endif
+
+static PyObject *
+moduleinit()
 {
-	PyObject *zfs_ioctl = Py_InitModule("zfs.ioctl", zfsmethods);
-	PyObject *zfs_util = PyImport_ImportModule("zfs.util");
-	PyObject *devfile;
+	PyObject *zfs_ioctl, *zfs_util, *devfile;
+#if PY_MAJOR_VERSION >= 3
+	zfs_ioctl = PyModule_Create(&zfs_module);
+#else
+	zfs_ioctl = Py_InitModule("zfs.ioctl", zfsmethods);
+#endif
+	zfs_util = PyImport_ImportModule("zfs.util");
 
 	if (zfs_util == NULL)
-		return;
+		return (NULL);
 
 	ZFSError = PyObject_GetAttrString(zfs_util, "ZFSError");
 	devfile = PyObject_GetAttrString(zfs_util, "dev");
 	zfsdevfd = PyObject_AsFileDescriptor(devfile);
 
 	zfs_prop_init();
+
+	return (zfs_ioctl);
+}
+
+#if PY_MAJOR_VERSION >= 3
+PyMODINIT_FUNC
+PyInit_ioctl(void)
+{
+	return (moduleinit());
 }
+#else
+PyMODINIT_FUNC
+initioctl(void)
+{
+	(void) moduleinit();
+}
+#endif
diff --git a/usr/src/lib/pyzfs/common/mapfile-py3 b/usr/src/lib/pyzfs/common/mapfile-py3
new file mode 100644
index 0000000000..a45e760e01
--- /dev/null
+++ b/usr/src/lib/pyzfs/common/mapfile-py3
@@ -0,0 +1,47 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+#
+# Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+#
+
+#
+# MAPFILE HEADER START
+#
+# WARNING:  STOP NOW.  DO NOT MODIFY THIS FILE.
+# Object versioning must comply with the rules detailed in
+#
+#	usr/src/lib/README.mapfiles
+#
+# You should not be making modifications here until you've read the most current
+# copy of that file. If you need help, contact a gatekeeper for guidance.
+#
+# MAPFILE HEADER END
+#
+
+$mapfile_version 2
+
+SYMBOL_VERSION SUNWprivate {
+    global:
+	PyInit_ioctl;
+    local:
+	*;
+};
diff --git a/usr/src/lib/pyzfs/common/table.py b/usr/src/lib/pyzfs/common/table.py
index 87aab1d6f1..173f06b5a0 100644
--- a/usr/src/lib/pyzfs/common/table.py
+++ b/usr/src/lib/pyzfs/common/table.py
@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 import zfs.util
@@ -34,7 +35,7 @@ class Table:
 		self.rjustfields = rjustfields
 		self.maxfieldlen = dict.fromkeys(fields, 0)
 		self.lines = list()
-	
+
 	def __updatemax(self, k, v):
 		self.maxfieldlen[k] = max(self.maxfieldlen.get(k, None), v)
 
@@ -46,6 +47,8 @@ class Table:
 			v = str(values[f])
 			va.append(v)
 			self.__updatemax(f, len(v))
+		if sortkey == None:
+			sortkey = []
 		self.lines.append((sortkey, va))
 
 	def printme(self, headers=True):
diff --git a/usr/src/lib/pyzfs/common/userspace.py b/usr/src/lib/pyzfs/common/userspace.py
index 79355e1e23..89cc18b241 100644
--- a/usr/src/lib/pyzfs/common/userspace.py
+++ b/usr/src/lib/pyzfs/common/userspace.py
@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """This module implements the "zfs userspace" and "zfs groupspace" subcommands.
@@ -115,7 +116,7 @@ def process_one_raw(acct, options, prop, elem):
 		except KeyError:
 			pass;
 	key = (isgroup, domain, rid)
-		
+
 	try:
 		v = acct[key]
 	except KeyError:
@@ -241,6 +242,6 @@ def do_userspace():
 		return l
 
 	t = zfs.table.Table(options.fields, rjustfields)
-	for val in acct.itervalues():
+	for val in acct.values():
 		t.addline(cmpkey(val), val)
 	t.printme(not options.noheaders)
diff --git a/usr/src/lib/pyzfs/common/util.py b/usr/src/lib/pyzfs/common/util.py
index cfc21ac57c..f384501c11 100644
--- a/usr/src/lib/pyzfs/common/util.py
+++ b/usr/src/lib/pyzfs/common/util.py
@@ -20,6 +20,7 @@
 # CDDL HEADER END
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 """This module provides utility functions for ZFS.
@@ -54,11 +55,11 @@ def default_repr(self):
 		return "<%s %s>" % \
 		    (self.__class__.__name__, repr(self.__dict__))
 
-class ZFSError(StandardError):
+class ZFSError(Exception):
 	"""This exception class represents a potentially user-visible
 	ZFS error.  If uncaught, it will be printed and the process will
 	exit with exit code 1.
-	
+
 	errno -- the error number (eg, from ioctl(2))."""
 
 	__slots__ = "why", "task", "errno"
diff --git a/usr/src/lib/pyzfs/i386/Makefile b/usr/src/lib/pyzfs/i386/Makefile
index 4ce880c2ae..82d8120e69 100644
--- a/usr/src/lib/pyzfs/i386/Makefile
+++ b/usr/src/lib/pyzfs/i386/Makefile
@@ -20,8 +20,10 @@
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 include ../Makefile.com
 
-install: all $(ROOTLIBS)
+all: $(PBOJECT) $(LIBS)
+install: all $(ROOTPYZFSFILES) $(ROOTLIBS)
diff --git a/usr/src/lib/pyzfs/py3/Makefile b/usr/src/lib/pyzfs/py3/Makefile
new file mode 100644
index 0000000000..27e04f084e
--- /dev/null
+++ b/usr/src/lib/pyzfs/py3/Makefile
@@ -0,0 +1,39 @@
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+
+#
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+#
+
+include ../Makefile.com
+include ../../Makefile.lib.64
+
+PYTHON = $(PYTHON3)
+PYTHON_VERSION = $(PYTHON3_VERSION)
+PYTHON_SUFFIX = $(PYTHON3_SUFFIX)
+
+LDLIBS64 += -L$(ADJUNCT_PROTO)/usr/lib/$(MACH64)
+MAPFILES= $(SRCDIR)/mapfile-py3
+ROOTPYZFSFILES=	$(PYSRCS:%=$(ROOTLIBDIR)/%)
+$(ROOTLIBDIR)/$(DYNLIB) := FILEMODE= 755
+
+all: $(LIBS)
+install: all $(ROOTLIBS) $(ROOTPYZFSFILES)
+
+$(ROOTLIBDIR)/%.so: %.so
+	$(INS.file)
+
+$(ROOTLIBDIR)/%: ../common/%
+	$(INS.pyfile)
+	$(PYTHON) -mpy_compile $@
+