9979 Support python3 for in-gate tools

Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Peter Tribble <peter.tribble@gmail.com>
Reviewed by: Alexander Pyhalov <apyhalov@gmail.com>
Reviewed by: Andrew Stormont <andyjstormont@gmail.com>
Approved by: Dan McDonald <danmcd@joyent.com>

12 files changed, 203 insertions, 65 deletions

diff --git a/usr/src/tools/onbld/Checks/Cddl.py b/usr/src/tools/onbld/Checks/Cddl.py
index 1f5f99f953..e2bbf09c02 100644
--- a/usr/src/tools/onbld/Checks/Cddl.py
+++ b/usr/src/tools/onbld/Checks/Cddl.py
@@ -26,11 +26,14 @@ CDDL HEADER END
 # Use is subject to license terms.
 #
 
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+
 #
 # Check that source files contain a valid CDDL block
 #
 
-import sys, CmtBlk
+import sys
+from onbld.Checks import CmtBlk
 
 # scmtest has a test for cddlchk that depends on the variable
 # Cddl.CmntChrs. However, that variable has been refactored into
@@ -43,7 +46,7 @@ CmntChrs = CmtBlk.CmntChrs
 # The CDDL string above contains the block guards so that the text will
 # be tested by cddlchk. However, we don't want to include the initial
 # \n or the block guards in the text passed in.
-# 
+#
 CDDL = CDDL.splitlines()[3:-2]
 
 def cddlchk(fh, filename=None, lenient=False, verbose=False, output=sys.stderr):
diff --git a/usr/src/tools/onbld/Checks/CmtBlk.py b/usr/src/tools/onbld/Checks/CmtBlk.py
index 2f3d29fa79..57eb5704c1 100644
--- a/usr/src/tools/onbld/Checks/CmtBlk.py
+++ b/usr/src/tools/onbld/Checks/CmtBlk.py
@@ -23,6 +23,7 @@
 
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -73,7 +74,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None,
 	for line in fh:
 		line = line.rstrip('\r\n')
 		lineno += 1
-		
+
 		if StartRE.search(line):
 			in_cmt = True
 			lic.append(line)
@@ -110,7 +111,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None,
 	for b in blocks:
 		try:
 			checkblock(b, full_text)
-		except CmtBlkError, e:
+		except CmtBlkError as e:
 			ret = 1
 			output.write(
 				"%s: %d: Error: Invalid line in %s block:\n"
@@ -120,7 +121,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None,
 				"    '%s'\n" % (filename, e.lineno, blk_name,
 						e.shouldbe, e.seen))
 			break
-		
+
 	if verbose and not ret:
 		output.write("%s: Valid %s block\n" %
 			     (filename, blk_name))
diff --git a/usr/src/tools/onbld/Checks/Comments.py b/usr/src/tools/onbld/Checks/Comments.py
index daf6aa47d9..4b4706618b 100644
--- a/usr/src/tools/onbld/Checks/Comments.py
+++ b/usr/src/tools/onbld/Checks/Comments.py
@@ -26,12 +26,13 @@
 #
 
 # Copyright 2007, 2010 Richard Lowe
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 
 #
 # Check delta comments:
-# 	- Have the correct form.
-# 	- Have a synopsis matching that of the bug
-# 	- Appear only once.
+#	- Have the correct form.
+#	- Have a synopsis matching that of the bug
+#	- Appear only once.
 #
 
 import re, sys
@@ -113,9 +114,9 @@ def comchk(comments, check_db=True, output=sys.stderr):
 
 	if len(bugs) > 0 and check_db:
 		bugdb = BugDB()
-		results = bugdb.lookup(bugs.keys())
+		results = bugdb.lookup(list(bugs.keys()))
 
-	for crid, insts in bugs.iteritems():
+	for crid, insts in bugs.items():
 		if len(insts) > 1:
 			errors['dup'].append(crid)
 
diff --git a/usr/src/tools/onbld/Checks/DbLookups.py b/usr/src/tools/onbld/Checks/DbLookups.py
index 11fd4185be..324cc58209 100644
--- a/usr/src/tools/onbld/Checks/DbLookups.py
+++ b/usr/src/tools/onbld/Checks/DbLookups.py
@@ -25,6 +25,7 @@
 #
 
 # Copyright 2010, Richard Lowe
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 
 #
 # Various database lookup classes/methods, i.e.:
@@ -33,10 +34,13 @@
 #     * redmine (illumos.org)
 #
 
-import htmllib
 import re
-import urllib
-import urllib2
+try:
+	from urllib.request import urlopen, Request
+	from urllib.error import HTTPError
+except ImportError:
+	# Python 2
+	from urllib2 import Request, urlopen, HTTPError
 
 try:				# Python >= 2.5
 	from xml.etree import ElementTree
@@ -72,17 +76,17 @@ class BugDB(object):
 		"""
 		for database in priority:
 			if database not in self.VALID_DBS:
-				raise BugDBException, database
+				raise BugDBException(database)
 		self.__priority = priority
 
 
 	def __illbug(self, cr):
 		url = "http://illumos.org/issues/%s.xml" % cr
-		req = urllib2.Request(url)
+		req = Request(url)
 
 		try:
-			data = urllib2.urlopen(req)
-		except urllib2.HTTPError, e:
+			data = urlopen(req)
+		except HTTPError as e:
 			if e.code == 404:
 				raise NonExistentBug(cr)
 			else:
diff --git a/usr/src/tools/onbld/Checks/Keywords.py b/usr/src/tools/onbld/Checks/Keywords.py
index 5c374c3abb..cd4dda0b69 100644
--- a/usr/src/tools/onbld/Checks/Keywords.py
+++ b/usr/src/tools/onbld/Checks/Keywords.py
@@ -24,8 +24,8 @@
 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# ident	"%Z%%M%	%I%	%E% SMI"
-#
+
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 
 #
 # Mercurial (lack of) keyword checks
@@ -39,7 +39,7 @@ ident = re.compile(r'((\%Z\%(\%M\%)\s+\%I\%|\%W\%)\s+\%E\% SMI)')
 #
 # Absolutely anything that appears to be an SCCS keyword.
 # It's impossible to programatically differentiate between these
-# and other, legitimate, uses of matching strings. 
+# and other, legitimate, uses of matching strings.
 #
 anykword = re.compile(r'%[A-ILMP-UWYZ]%')
 
@@ -56,14 +56,14 @@ def keywords(fh, filename=None, lenient=False, verbose=False,
 
     ret = 0
     lineno = 0
-    
+
     for line in fh:
         line = line.rstrip('\r\n')
         lineno += 1
-        
+
         if lenient and ident.search(line):
             continue
-        
+
         match = anykword.findall(line)
         if match:
             ret = 1
diff --git a/usr/src/tools/onbld/Checks/Makefile b/usr/src/tools/onbld/Checks/Makefile
index bca61be9ee..b6f18bdf85 100644
--- a/usr/src/tools/onbld/Checks/Makefile
+++ b/usr/src/tools/onbld/Checks/Makefile
@@ -23,40 +23,24 @@
 # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 #
 
-# Copyright 2010, Richard Lowe
-# Copyright 2014 Garrett D'Amore <garrett@damore.org>
-# Copyright 2016, Joyent, Inc.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 
-include $(SRC)/Makefile.master
 include ../../Makefile.tools
 
-PYSRCS = \
-	CStyle.py	\
-	Cddl.py		\
-	CmtBlk.py	\
-	Comments.py	\
-	Copyright.py	\
-	DbLookups.py	\
-	HdrChk.py	\
-	JStyle.py	\
-	Keywords.py	\
-	ManLint.py	\
-	Mapfile.py	\
-	ProcessCheck.py \
-	SpellCheck.py \
-	WsCheck.py \
-	__init__.py
+SUBDIRS=
+$(BUILDPY2TOOLS)SUBDIRS += py2
+$(BUILDPY3TOOLS)SUBDIRS += py3
 
-PYOBJS =	$(PYSRCS:%.py=%.pyc)
-PYTOPDIR =	$(ROOTONBLDLIB)
-PYMODDIR =	onbld/Checks
+all :=		TARGET= all
+install :=	TARGET= install
+clean :=	TARGET= clean
+clobber :=	TARGET= clobber
+lint :=		TARGET= lint
 
-include ../../Makefile.python
+all install clean clobber lint: $(SUBDIRS)
 
-all: $(PYVERSOJBS)
+$(SUBDIRS): FRC
+	@cd $@; pwd; $(MAKE) $(TARGET)
 
-install: all $(ROOTPYFILES)
+FRC:
 
-clean:
-
-clobber: clean pyclobber
diff --git a/usr/src/tools/onbld/Checks/Makefile.com b/usr/src/tools/onbld/Checks/Makefile.com
new file mode 100644
index 0000000000..e8872e8ca8
--- /dev/null
+++ b/usr/src/tools/onbld/Checks/Makefile.com
@@ -0,0 +1,53 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+#
+
+# Copyright 2010, Richard Lowe
+# Copyright 2014 Garrett D'Amore <garrett@damore.org>
+# Copyright 2016, Joyent, Inc.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+
+include $(SRC)/Makefile.master
+include ../../../Makefile.tools
+
+PYTOPDIR =	$(ROOTONBLDLIB)
+PYMODDIR =	onbld/Checks
+
+PYSRCS = \
+	CStyle.py	\
+	Cddl.py		\
+	CmtBlk.py	\
+	Comments.py	\
+	Copyright.py	\
+	DbLookups.py	\
+	HdrChk.py	\
+	JStyle.py	\
+	Keywords.py	\
+	ManLint.py	\
+	Mapfile.py	\
+	ProcessCheck.py	\
+	SpellCheck.py	\
+	WsCheck.py	\
+	__init__.py
+
diff --git a/usr/src/tools/onbld/Checks/Mapfile.py b/usr/src/tools/onbld/Checks/Mapfile.py
index 2a8cb74aed..5b40d26005 100644
--- a/usr/src/tools/onbld/Checks/Mapfile.py
+++ b/usr/src/tools/onbld/Checks/Mapfile.py
@@ -22,6 +22,7 @@
 
 #
 # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 #
@@ -38,7 +39,8 @@ You should not be making modifications here until you've read the most current
 copy of that file. If you need help, contact a gatekeeper for guidance.
 '''
 
-import re, sys, CmtBlk
+import re, sys
+from onbld.Checks import CmtBlk
 
 MAPFILE = MAPFILE.splitlines()[1:]		# Don't include initial \n
 
@@ -51,9 +53,9 @@ def mapfilechk(fh, filename=None, verbose=False, output=sys.stderr):
 	# Verify that the mapfile is using version 2 syntax. Read and discard
 	# comment and empty lines until the first non-empty line is seen.
 	# This line must be '$mapfile_version 2'.
-        CmtRE = re.compile(r'#.*$')
-        LeadingWSRE = re.compile(r'^\s+')
-        VersionRE = re.compile(r'^\$mapfile_version\s+2\s*$')
+	CmtRE = re.compile(r'#.*$')
+	LeadingWSRE = re.compile(r'^\s+')
+	VersionRE = re.compile(r'^\$mapfile_version\s+2\s*$')
 	for line in fh:
 		line = CmtRE.sub(r'', line)
 		line = LeadingWSRE.sub(r'', line)
@@ -68,13 +70,12 @@ def mapfilechk(fh, filename=None, verbose=False, output=sys.stderr):
 
 		# We have verified version 2 syntax. Exit the loop
 		break
-		
 
 	# If the mapfile contains a SYMBOL_VERSION directive, the file
 	# must include a copy of the MAPFILE warning comment above. The
 	# comment is specific to symbol versioning, so we don't harrass
 	# the authors of mapfiles used exclusively for other purposes.
-        SymVerRE = re.compile(r'^\s*symbol_version\s+', re.IGNORECASE)
+	SymVerRE = re.compile(r'^\s*symbol_version\s+', re.IGNORECASE)
 	for line in fh:
 		# If we find a SYMBOL_VERSION, then verify that the comment
 		# is present. The comment usually precedes the mapfile_version
diff --git a/usr/src/tools/onbld/Checks/ProcessCheck.py b/usr/src/tools/onbld/Checks/ProcessCheck.py
index 52ebabd002..1d3bf9df07 100644
--- a/usr/src/tools/onbld/Checks/ProcessCheck.py
+++ b/usr/src/tools/onbld/Checks/ProcessCheck.py
@@ -23,8 +23,8 @@
 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# ident	"%Z%%M%	%I%	%E% SMI"
-#
+
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 
 #
 # Wrap a command-line check tool in a pythonic API
@@ -46,8 +46,8 @@ def processcheck(command, args, inpt, output):
 	# don't deadlock with the child if both pipes fill.
 	#
 	try:
-		tmpfile = tempfile.TemporaryFile(prefix=command)
-	except EnvironmentError, e:
+		tmpfile = tempfile.TemporaryFile(prefix=command, mode="w+b")
+	except EnvironmentError as e:
 		output.write("Could not create temporary file: %s\n" % e)
 		return (3, None)
 
@@ -55,7 +55,7 @@ def processcheck(command, args, inpt, output):
 		p = subprocess.Popen([command] + args,
 				     stdin=subprocess.PIPE, stdout=tmpfile,
 				     stderr=subprocess.STDOUT, close_fds=False)
-	except OSError, e:
+	except OSError as e:
 		output.write("Could not execute %s: %s\n" % (command, e))
 		return (3, None)
 
diff --git a/usr/src/tools/onbld/Checks/SpellCheck.py b/usr/src/tools/onbld/Checks/SpellCheck.py
index ccf3d57e5e..bb89306cc7 100644
--- a/usr/src/tools/onbld/Checks/SpellCheck.py
+++ b/usr/src/tools/onbld/Checks/SpellCheck.py
@@ -21,6 +21,7 @@
 
 #
 # Copyright 2016 Joyent, Inc.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 #
 
 import re, sys
@@ -255,12 +256,12 @@ alternates = {
 misspellingREs = []
 alternateREs = []
 
-for misspelling, correct in misspellings.iteritems():
+for misspelling, correct in misspellings.items():
 	regex = re.compile(r'\b%s\b' % (misspelling), re.IGNORECASE)
 	entry = (regex, misspelling, correct)
 	misspellingREs.append(entry)
 
-for alternate, correct in alternates.iteritems():
+for alternate, correct in alternates.items():
 	regex = re.compile(r'\b%s\b' % (alternate), re.IGNORECASE)
 	entry = (regex, alternate, correct)
 	alternateREs.append(entry)
diff --git a/usr/src/tools/onbld/Checks/py2/Makefile b/usr/src/tools/onbld/Checks/py2/Makefile
new file mode 100644
index 0000000000..40b2b41255
--- /dev/null
+++ b/usr/src/tools/onbld/Checks/py2/Makefile
@@ -0,0 +1,45 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+#
+
+# Copyright 2010, Richard Lowe
+# Copyright 2014 Garrett D'Amore <garrett@damore.org>
+# Copyright 2016, Joyent, Inc.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+
+include ../Makefile.com
+
+PYVER = $(PYTHON_VERSION)
+PYPKGVERS = $(PYTHON_PKGVERS)
+PYOBJS =	$(PYSRCS:%.py=%.pyc)
+
+include ../../../Makefile.python
+
+all: $(PYVERSOJBS)
+
+install: all $(ROOTPYFILES)
+
+clean:
+
+clobber: clean pyclobber
diff --git a/usr/src/tools/onbld/Checks/py3/Makefile b/usr/src/tools/onbld/Checks/py3/Makefile
new file mode 100644
index 0000000000..0bf31188b8
--- /dev/null
+++ b/usr/src/tools/onbld/Checks/py3/Makefile
@@ -0,0 +1,45 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+#
+
+# Copyright 2010, Richard Lowe
+# Copyright 2014 Garrett D'Amore <garrett@damore.org>
+# Copyright 2016, Joyent, Inc.
+# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
+
+include ../Makefile.com
+
+PYVER = $(PYTHON3_VERSION)
+PYPKGVERS = $(PYTHON3_PKGVERS)
+PYOBJS = $(PYSRCS:%.py=__pycache__/%.cpython$(PYTHON3_PKGVERS).pyc)
+
+include ../../../Makefile.python
+
+all:
+install: $(ROOTPYFILES)
+
+clean:
+
+clobber: clean pyclobber
+