diff options
| author | Andy Fiddaman <omnios@citrus-it.co.uk> | 2018-11-15 10:17:46 +0000 |
|---|---|---|
| committer | Joshua M. Clulow <josh@sysmgr.org> | 2020-04-08 12:30:39 -0700 |
| commit | b9219c8a2cb8887f04ec2b394b225390a6e624ae (patch) | |
| tree | aae332b6fdc157b28458f10b1627f389dbfe59b2 | |
| parent | 5bbe7d1e9f312861ee789d5a8dc75436fdc8bed2 (diff) | |
| download | illumos-gate-b9219c8a2cb8887f04ec2b394b225390a6e624ae.tar.gz | |
9979 Support python3 for in-gate tools
Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Peter Tribble <peter.tribble@gmail.com>
Reviewed by: Alexander Pyhalov <apyhalov@gmail.com>
Reviewed by: Andrew Stormont <andyjstormont@gmail.com>
Approved by: Dan McDonald <danmcd@joyent.com>
34 files changed, 1040 insertions, 480 deletions
diff --git a/usr/src/Makefile.master b/usr/src/Makefile.master index a63c36d53f..9a1ede9e99 100644 --- a/usr/src/Makefile.master +++ b/usr/src/Makefile.master @@ -199,6 +199,8 @@ PYTHON3_VERSION= 3.5 PYTHON3_PKGVERS= -35 PYTHON3_SUFFIX= m PYTHON3= /usr/bin/python$(PYTHON3_VERSION) +$(BUILDPY3TOOLS)TOOLS_PYTHON= $(PYTHON3) +$(BUILDPY2TOOLS)TOOLS_PYTHON= $(PYTHON) SORT= /usr/bin/sort TR= /usr/bin/tr TOUCH= /usr/bin/touch @@ -275,7 +277,10 @@ PYSHEBANG= $(PYTHON) # files we must make certain to not adjust the mtime of the source # (.py) file. # -INS.pyfile= $(RM) $@; $(SED) -e "1s:^\#!@PYTHON@:\#!$(PYSHEBANG):" < $< > $@; $(CHMOD) $(FILEMODE) $@; $(TOUCH) -r $< $@ +INS.pyfile= $(RM) $@; $(SED) \ + -e "1s:^\#!@PYTHON@:\#!$(PYSHEBANG):" \ + -e "1s:^\#!@TOOLS_PYTHON@:\#!$(TOOLS_PYTHON):" \ + < $< > $@; $(CHMOD) $(FILEMODE) $@; $(TOUCH) -r $< $@ # MACH must be set in the shell environment per uname -p on the build host # More specific architecture variables should be set in lower makefiles. @@ -1125,7 +1130,10 @@ PKGPUBLISHER_NONREDIST= on-extra $(CHMOD) +x $@ .py: - $(RM) $@; $(SED) -e "1s:^\#!@PYTHON@:\#!$(PYSHEBANG):" < $< > $@; $(CHMOD) +x $@ + $(RM) $@; $(SED) \ + -e "1s:^\#!@PYTHON@:\#!$(PYSHEBANG):" \ + -e "1s:^\#!@TOOLS_PYTHON@:\#!$(TOOLS_PYTHON):" \ + < $< > $@; $(CHMOD) +x $@ .py.pyc: $(RM) $@ diff --git a/usr/src/data/locale/tools/mkwidths.py b/usr/src/data/locale/tools/mkwidths.py index 19a587419d..39ec96a1a0 100644 --- a/usr/src/data/locale/tools/mkwidths.py +++ b/usr/src/data/locale/tools/mkwidths.py @@ -16,6 +16,10 @@ Scratch script to produce the widths.cm content from the widths text files. It converts numeric unicode to symbolic forms. """ +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +from __future__ import print_function + SYMBOLS = {} @@ -66,13 +70,13 @@ def do_width_file(width, filename): sym = SYMBOLS.get(key, None) if sym == None: continue - print "%s\t%d" % (sym, width) + print("%s\t%d" % (sym, width)) vals = vals[2:] if __name__ == "__main__": - print "WIDTH" + print("WIDTH") load_utf8() do_width_file(0, "widths-0.txt") do_width_file(2, "widths-2.txt") - print "END WIDTH" + print("END WIDTH") diff --git a/usr/src/pkg/Makefile b/usr/src/pkg/Makefile index cacaa22aca..3ac8594165 100644 --- a/usr/src/pkg/Makefile +++ b/usr/src/pkg/Makefile @@ -195,6 +195,8 @@ PKGMOG_DEFINES= \ PYTHON3_PKGVERS=$(PYTHON3_PKGVERS) \ python2_ONLY=$(BUILDPY2) \ python3_ONLY=$(BUILDPY3) \ + python2tools_ONLY=$(BUILDPY2TOOLS) \ + python3tools_ONLY=$(BUILDPY3TOOLS) \ JAVA_8_ONLY=$(JAVA_8_ONLY) \ JAVA_7_ONLY=$(JAVA_7_ONLY) diff --git a/usr/src/pkg/manifests/developer-build-onbld.mf b/usr/src/pkg/manifests/developer-build-onbld.mf index 49863c8e63..dd80e1f0a4 100644 --- a/usr/src/pkg/manifests/developer-build-onbld.mf +++ b/usr/src/pkg/manifests/developer-build-onbld.mf @@ -26,6 +26,7 @@ # Copyright 2014 Garrett D'Amore <garrett@damore.org> # Copyright (c) 2014, Joyent, Inc. # Copyright 2016 Toomas Soome <tsoome@me.com> +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # set name=pkg.fmri value=pkg:/developer/build/onbld@$(PKGVERS) @@ -51,10 +52,22 @@ dir path=opt/onbld/lib dir path=opt/onbld/lib/$(ARCH) dir path=opt/onbld/lib/$(ARCH)/64 dir path=opt/onbld/lib/perl -dir path=opt/onbld/lib/python$(PYTHON_VERSION) -dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld -dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks -dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm +$(python3tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON3_VERSION) +$(python3tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld +$(python3tools_ONLY)dir \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks +$(python3tools_ONLY)dir \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__ +$(python3tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm +$(python3tools_ONLY)dir \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm/__pycache__ +$(python3tools_ONLY)dir \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/__pycache__ +$(python2tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON_VERSION) +$(python2tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld +$(python2tools_ONLY)dir \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks +$(python2tools_ONLY)dir path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm dir path=opt/onbld/man dir path=opt/onbld/man/man1onbld dir path=opt/onbld/share @@ -135,73 +148,215 @@ file path=opt/onbld/lib/$(ARCH)/libdwarf.so.1 file path=opt/onbld/lib/$(ARCH)/libmakestate.so.1 file path=opt/onbld/lib/perl/onbld_elfmod.pm file path=opt/onbld/lib/perl/onbld_elfmod_vertype.pm -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/CStyle.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/Cddl.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.py mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/CmtBlk.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/Comments.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/Copyright.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/DbLookups.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/HdrChk.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/JStyle.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/Keywords.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/ManLint.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/Mapfile.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/ProcessCheck.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/SpellCheck.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/WsCheck.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__init__.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/CStyle.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/Cddl.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/CmtBlk.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/Comments.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/Copyright.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/DbLookups.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/HdrChk.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/JStyle.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/Keywords.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/ManLint.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/WsCheck.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/Mapfile.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/WsCheck.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/ProcessCheck.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/SpellCheck.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/WsCheck.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.py mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.pyc mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.py \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__/__init__.cpython$(PYTHON3_PKGVERS).pyc \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.pyc \ +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm/Ignore.py mode=0444 +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm/__init__.py \ mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.py mode=0444 -file path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.pyc mode=0444 +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm/__pycache__/Ignore.cpython$(PYTHON3_PKGVERS).pyc \ + mode=0444 +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/Scm/__pycache__/__init__.cpython$(PYTHON3_PKGVERS).pyc \ + mode=0444 +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/__init__.py mode=0444 +$(python3tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON3_VERSION)/onbld/__pycache__/__init__.cpython$(PYTHON3_PKGVERS).pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CStyle.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.py mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Cddl.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/CmtBlk.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Comments.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Copyright.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/DbLookups.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/HdrChk.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/JStyle.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Keywords.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ManLint.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/Mapfile.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/ProcessCheck.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/SpellCheck.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/WsCheck.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/WsCheck.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Checks/__init__.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.py mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/Ignore.pyc mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.py \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/Scm/__init__.pyc \ + mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.py mode=0444 +$(python2tools_ONLY)file \ + path=opt/onbld/lib/python$(PYTHON_VERSION)/onbld/__init__.pyc mode=0444 file path=opt/onbld/man/man1onbld/Install.1onbld file path=opt/onbld/man/man1onbld/bldenv.1onbld file path=opt/onbld/man/man1onbld/bringovercheck.1onbld @@ -243,7 +398,8 @@ license usr/src/tools/ctf/dwarf/THIRDPARTYLICENSE \ license=usr/src/tools/ctf/dwarf/THIRDPARTYLICENSE link path=opt/onbld/bin/$(ARCH)/dmake target=make link path=opt/onbld/bin/git-nits target=git-pbchk -link path=opt/onbld/lib/python target=python$(PYTHON_VERSION) +$(python2tools_ONLY)link path=opt/onbld/lib/python \ + target=python$(PYTHON_VERSION) link path=opt/onbld/man/man1onbld/git-nits.1onbld target=git-pbchk.1onbld # webrev(1) requires ps2pdf depend fmri=print/filter/ghostscript type=require diff --git a/usr/src/tools/Makefile b/usr/src/tools/Makefile index 95c562883a..26308b1eb7 100644 --- a/usr/src/tools/Makefile +++ b/usr/src/tools/Makefile @@ -24,6 +24,8 @@ # Copyright 2014 Garrett D'Amore <garrett@damore.org> # Copyright 2016 Toomas Soome <tsoome@me.com> # Copyright (c) 2016, Chris Fraire <cfraire@me.com>. +# Copyright (c) 2019, Joyent, Inc. +# Copyright 2019 OmniOS Community Edition (OmniOSce) Association. # include ../Makefile.master @@ -103,10 +105,6 @@ ROOTDIRS= \ $(ROOTONBLD)/lib/$(MACH) \ $(ROOTONBLD)/lib/$(MACH)/64 \ $(ROOTONBLD)/lib/perl \ - $(ROOTONBLD)/lib/python$(PYTHON_VERSION) \ - $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld \ - $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Checks \ - $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Scm \ $(ROOTONBLD)/env \ $(ROOTONBLD)/etc \ $(ROOTONBLD)/etc/exception_lists \ @@ -114,6 +112,21 @@ ROOTDIRS= \ $(ROOTONBLD)/man \ $(ROOTONBLD)/man/man1onbld +$(BUILDPY2TOOLS)ROOTDIRS += \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION) \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Checks \ + $(ROOTONBLD)/lib/python$(PYTHON_VERSION)/onbld/Scm + +$(BUILDPY3TOOLS)ROOTDIRS += \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION) \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld/__pycache__ \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld/Checks \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld/Checks/__pycache__ \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld/Scm \ + $(ROOTONBLD)/lib/python$(PYTHON3_VERSION)/onbld/Scm/__pycache__ + all := TARGET= install install := TARGET= install clean := TARGET= clean @@ -131,7 +144,7 @@ DOROOTDIRS= $(ROOTDIRS) clobber:= DOROOTDIRS= clean:= DOROOTDIRS= -DOROOTONBLDLIBPY= $(ROOTONBLDLIBPY) +$(BUILDPY2TOOLS)DOROOTONBLDLIBPY= $(ROOTONBLDLIBPY) clobber:= DOROOTONBLDLIBPY= clean:= DOROOTONBLDLIBPY= diff --git a/usr/src/tools/Makefile.python b/usr/src/tools/Makefile.python index 147ef2b05c..63df6e7a36 100644 --- a/usr/src/tools/Makefile.python +++ b/usr/src/tools/Makefile.python @@ -21,6 +21,7 @@ # # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -31,12 +32,12 @@ # # PYSRCS - List of python source files, these are also delivered as # build product. -# +# # PYOBJS - List of compiled python (.pyc) files, with no directory prefix -# +# # PYTOPDIR - Absolute (including $(ROOT)) path to which files will # be installed, up until the version specific component. -# +# # PYMODDIR - Relative path to which files will be installed, below # the version specific component. # @@ -63,32 +64,46 @@ PYFILES = $(PYSRCS) $(PYOBJS) -ROOTPYDIR = $(PYTOPDIR)/python$(PYTHON_VERSION)/$(PYMODDIR) +ROOTPYDIR = $(PYTOPDIR)/python$(PYVER)/$(PYMODDIR) ROOTPYFILES = $(PYFILES:%=$(ROOTPYDIR)/%) $(ROOTPYFILES) := FILEMODE = 0444 -PYVERSDIRS = python$(PYTHON_VERSION) +PYVERSDIRS = python$(PYVER) -PYVERSOBJS = $(PYOBJS:%=python$(PYTHON_VERSION)/%) +PYVERSOBJS = $(PYOBJS:%=python$(PYVER)/%) CLOBBERFILES += $(PYVERSOBJS) CLOBBERDIRS += $(PYVERSDIRS) .KEEP_STATE: -python$(PYTHON_VERSION)/%.pyc: %.py +$(ROOTPYDIR)/%.py: ../%.py + $(INS.pyfile) + +pyclobber: + $(RM) $(CLOBBERFILES) + $(RM) -rf $(CLOBBERDIRS) + +# Python 2 module compilation + +python$(PYVER)/%.pyc: ../%.py @[ -d $(@D) ] || mkdir $(@D) $(RM) $@ $(PYTHON) -mpy_compile $< - $(MV) $(*).pyc $@ + $(MV) ../$(*).pyc $@ -$(ROOTPYDIR)/%.pyc: python$(PYTHON_VERSION)/%.pyc +$(ROOTPYDIR)/%.pyc: python$(PYVER)/%.pyc $(INS.pyfile) -$(ROOTPYDIR)/%.py: %.py +# Python 3 module compilation + +python$(PYVER)/__pycache__/%.cpython$(PYPKGVERS).pyc: ../%.py + @[ -d $(@D) ] || mkdir -p $(@D) + $(RM) $@ + $(PYTHON3) -mpy_compile $< + $(MV) ../__pycache__/$(*).cpython$(PYPKGVERS).pyc $@ + +$(ROOTPYDIR)/__pycache__/%.pyc: python$(PYVER)/__pycache__/%.pyc $(INS.pyfile) -pyclobber: - $(RM) $(CLOBBERFILES) - $(RM) -rf $(CLOBBERDIRS) diff --git a/usr/src/tools/onbld/Checks/Cddl.py b/usr/src/tools/onbld/Checks/Cddl.py index 1f5f99f953..e2bbf09c02 100644 --- a/usr/src/tools/onbld/Checks/Cddl.py +++ b/usr/src/tools/onbld/Checks/Cddl.py @@ -26,11 +26,14 @@ CDDL HEADER END # Use is subject to license terms. # +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + # # Check that source files contain a valid CDDL block # -import sys, CmtBlk +import sys +from onbld.Checks import CmtBlk # scmtest has a test for cddlchk that depends on the variable # Cddl.CmntChrs. However, that variable has been refactored into @@ -43,7 +46,7 @@ CmntChrs = CmtBlk.CmntChrs # The CDDL string above contains the block guards so that the text will # be tested by cddlchk. However, we don't want to include the initial # \n or the block guards in the text passed in. -# +# CDDL = CDDL.splitlines()[3:-2] def cddlchk(fh, filename=None, lenient=False, verbose=False, output=sys.stderr): diff --git a/usr/src/tools/onbld/Checks/CmtBlk.py b/usr/src/tools/onbld/Checks/CmtBlk.py index 2f3d29fa79..57eb5704c1 100644 --- a/usr/src/tools/onbld/Checks/CmtBlk.py +++ b/usr/src/tools/onbld/Checks/CmtBlk.py @@ -23,6 +23,7 @@ # # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -73,7 +74,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None, for line in fh: line = line.rstrip('\r\n') lineno += 1 - + if StartRE.search(line): in_cmt = True lic.append(line) @@ -110,7 +111,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None, for b in blocks: try: checkblock(b, full_text) - except CmtBlkError, e: + except CmtBlkError as e: ret = 1 output.write( "%s: %d: Error: Invalid line in %s block:\n" @@ -120,7 +121,7 @@ def cmtblkchk(fh, blk_name, blk_text, filename=None, " '%s'\n" % (filename, e.lineno, blk_name, e.shouldbe, e.seen)) break - + if verbose and not ret: output.write("%s: Valid %s block\n" % (filename, blk_name)) diff --git a/usr/src/tools/onbld/Checks/Comments.py b/usr/src/tools/onbld/Checks/Comments.py index daf6aa47d9..4b4706618b 100644 --- a/usr/src/tools/onbld/Checks/Comments.py +++ b/usr/src/tools/onbld/Checks/Comments.py @@ -26,12 +26,13 @@ # # Copyright 2007, 2010 Richard Lowe +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # Check delta comments: -# - Have the correct form. -# - Have a synopsis matching that of the bug -# - Appear only once. +# - Have the correct form. +# - Have a synopsis matching that of the bug +# - Appear only once. # import re, sys @@ -113,9 +114,9 @@ def comchk(comments, check_db=True, output=sys.stderr): if len(bugs) > 0 and check_db: bugdb = BugDB() - results = bugdb.lookup(bugs.keys()) + results = bugdb.lookup(list(bugs.keys())) - for crid, insts in bugs.iteritems(): + for crid, insts in bugs.items(): if len(insts) > 1: errors['dup'].append(crid) diff --git a/usr/src/tools/onbld/Checks/DbLookups.py b/usr/src/tools/onbld/Checks/DbLookups.py index 11fd4185be..324cc58209 100644 --- a/usr/src/tools/onbld/Checks/DbLookups.py +++ b/usr/src/tools/onbld/Checks/DbLookups.py @@ -25,6 +25,7 @@ # # Copyright 2010, Richard Lowe +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # Various database lookup classes/methods, i.e.: @@ -33,10 +34,13 @@ # * redmine (illumos.org) # -import htmllib import re -import urllib -import urllib2 +try: + from urllib.request import urlopen, Request + from urllib.error import HTTPError +except ImportError: + # Python 2 + from urllib2 import Request, urlopen, HTTPError try: # Python >= 2.5 from xml.etree import ElementTree @@ -72,17 +76,17 @@ class BugDB(object): """ for database in priority: if database not in self.VALID_DBS: - raise BugDBException, database + raise BugDBException(database) self.__priority = priority def __illbug(self, cr): url = "http://illumos.org/issues/%s.xml" % cr - req = urllib2.Request(url) + req = Request(url) try: - data = urllib2.urlopen(req) - except urllib2.HTTPError, e: + data = urlopen(req) + except HTTPError as e: if e.code == 404: raise NonExistentBug(cr) else: diff --git a/usr/src/tools/onbld/Checks/Keywords.py b/usr/src/tools/onbld/Checks/Keywords.py index 5c374c3abb..cd4dda0b69 100644 --- a/usr/src/tools/onbld/Checks/Keywords.py +++ b/usr/src/tools/onbld/Checks/Keywords.py @@ -24,8 +24,8 @@ # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -# ident "%Z%%M% %I% %E% SMI" -# + +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # Mercurial (lack of) keyword checks @@ -39,7 +39,7 @@ ident = re.compile(r'((\%Z\%(\%M\%)\s+\%I\%|\%W\%)\s+\%E\% SMI)') # # Absolutely anything that appears to be an SCCS keyword. # It's impossible to programatically differentiate between these -# and other, legitimate, uses of matching strings. +# and other, legitimate, uses of matching strings. # anykword = re.compile(r'%[A-ILMP-UWYZ]%') @@ -56,14 +56,14 @@ def keywords(fh, filename=None, lenient=False, verbose=False, ret = 0 lineno = 0 - + for line in fh: line = line.rstrip('\r\n') lineno += 1 - + if lenient and ident.search(line): continue - + match = anykword.findall(line) if match: ret = 1 diff --git a/usr/src/tools/onbld/Checks/Makefile b/usr/src/tools/onbld/Checks/Makefile index bca61be9ee..b6f18bdf85 100644 --- a/usr/src/tools/onbld/Checks/Makefile +++ b/usr/src/tools/onbld/Checks/Makefile @@ -23,40 +23,24 @@ # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. # -# Copyright 2010, Richard Lowe -# Copyright 2014 Garrett D'Amore <garrett@damore.org> -# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. -include $(SRC)/Makefile.master include ../../Makefile.tools -PYSRCS = \ - CStyle.py \ - Cddl.py \ - CmtBlk.py \ - Comments.py \ - Copyright.py \ - DbLookups.py \ - HdrChk.py \ - JStyle.py \ - Keywords.py \ - ManLint.py \ - Mapfile.py \ - ProcessCheck.py \ - SpellCheck.py \ - WsCheck.py \ - __init__.py +SUBDIRS= +$(BUILDPY2TOOLS)SUBDIRS += py2 +$(BUILDPY3TOOLS)SUBDIRS += py3 -PYOBJS = $(PYSRCS:%.py=%.pyc) -PYTOPDIR = $(ROOTONBLDLIB) -PYMODDIR = onbld/Checks +all := TARGET= all +install := TARGET= install +clean := TARGET= clean +clobber := TARGET= clobber +lint := TARGET= lint -include ../../Makefile.python +all install clean clobber lint: $(SUBDIRS) -all: $(PYVERSOJBS) +$(SUBDIRS): FRC + @cd $@; pwd; $(MAKE) $(TARGET) -install: all $(ROOTPYFILES) +FRC: -clean: - -clobber: clean pyclobber diff --git a/usr/src/tools/onbld/Checks/Makefile.com b/usr/src/tools/onbld/Checks/Makefile.com new file mode 100644 index 0000000000..e8872e8ca8 --- /dev/null +++ b/usr/src/tools/onbld/Checks/Makefile.com @@ -0,0 +1,53 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include $(SRC)/Makefile.master +include ../../../Makefile.tools + +PYTOPDIR = $(ROOTONBLDLIB) +PYMODDIR = onbld/Checks + +PYSRCS = \ + CStyle.py \ + Cddl.py \ + CmtBlk.py \ + Comments.py \ + Copyright.py \ + DbLookups.py \ + HdrChk.py \ + JStyle.py \ + Keywords.py \ + ManLint.py \ + Mapfile.py \ + ProcessCheck.py \ + SpellCheck.py \ + WsCheck.py \ + __init__.py + diff --git a/usr/src/tools/onbld/Checks/Mapfile.py b/usr/src/tools/onbld/Checks/Mapfile.py index 2a8cb74aed..5b40d26005 100644 --- a/usr/src/tools/onbld/Checks/Mapfile.py +++ b/usr/src/tools/onbld/Checks/Mapfile.py @@ -22,6 +22,7 @@ # # Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -38,7 +39,8 @@ You should not be making modifications here until you've read the most current copy of that file. If you need help, contact a gatekeeper for guidance. ''' -import re, sys, CmtBlk +import re, sys +from onbld.Checks import CmtBlk MAPFILE = MAPFILE.splitlines()[1:] # Don't include initial \n @@ -51,9 +53,9 @@ def mapfilechk(fh, filename=None, verbose=False, output=sys.stderr): # Verify that the mapfile is using version 2 syntax. Read and discard # comment and empty lines until the first non-empty line is seen. # This line must be '$mapfile_version 2'. - CmtRE = re.compile(r'#.*$') - LeadingWSRE = re.compile(r'^\s+') - VersionRE = re.compile(r'^\$mapfile_version\s+2\s*$') + CmtRE = re.compile(r'#.*$') + LeadingWSRE = re.compile(r'^\s+') + VersionRE = re.compile(r'^\$mapfile_version\s+2\s*$') for line in fh: line = CmtRE.sub(r'', line) line = LeadingWSRE.sub(r'', line) @@ -68,13 +70,12 @@ def mapfilechk(fh, filename=None, verbose=False, output=sys.stderr): # We have verified version 2 syntax. Exit the loop break - # If the mapfile contains a SYMBOL_VERSION directive, the file # must include a copy of the MAPFILE warning comment above. The # comment is specific to symbol versioning, so we don't harrass # the authors of mapfiles used exclusively for other purposes. - SymVerRE = re.compile(r'^\s*symbol_version\s+', re.IGNORECASE) + SymVerRE = re.compile(r'^\s*symbol_version\s+', re.IGNORECASE) for line in fh: # If we find a SYMBOL_VERSION, then verify that the comment # is present. The comment usually precedes the mapfile_version diff --git a/usr/src/tools/onbld/Checks/ProcessCheck.py b/usr/src/tools/onbld/Checks/ProcessCheck.py index 52ebabd002..1d3bf9df07 100644 --- a/usr/src/tools/onbld/Checks/ProcessCheck.py +++ b/usr/src/tools/onbld/Checks/ProcessCheck.py @@ -23,8 +23,8 @@ # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -# ident "%Z%%M% %I% %E% SMI" -# + +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # Wrap a command-line check tool in a pythonic API @@ -46,8 +46,8 @@ def processcheck(command, args, inpt, output): # don't deadlock with the child if both pipes fill. # try: - tmpfile = tempfile.TemporaryFile(prefix=command) - except EnvironmentError, e: + tmpfile = tempfile.TemporaryFile(prefix=command, mode="w+b") + except EnvironmentError as e: output.write("Could not create temporary file: %s\n" % e) return (3, None) @@ -55,7 +55,7 @@ def processcheck(command, args, inpt, output): p = subprocess.Popen([command] + args, stdin=subprocess.PIPE, stdout=tmpfile, stderr=subprocess.STDOUT, close_fds=False) - except OSError, e: + except OSError as e: output.write("Could not execute %s: %s\n" % (command, e)) return (3, None) diff --git a/usr/src/tools/onbld/Checks/SpellCheck.py b/usr/src/tools/onbld/Checks/SpellCheck.py index ccf3d57e5e..bb89306cc7 100644 --- a/usr/src/tools/onbld/Checks/SpellCheck.py +++ b/usr/src/tools/onbld/Checks/SpellCheck.py @@ -21,6 +21,7 @@ # # Copyright 2016 Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # import re, sys @@ -255,12 +256,12 @@ alternates = { misspellingREs = [] alternateREs = [] -for misspelling, correct in misspellings.iteritems(): +for misspelling, correct in misspellings.items(): regex = re.compile(r'\b%s\b' % (misspelling), re.IGNORECASE) entry = (regex, misspelling, correct) misspellingREs.append(entry) -for alternate, correct in alternates.iteritems(): +for alternate, correct in alternates.items(): regex = re.compile(r'\b%s\b' % (alternate), re.IGNORECASE) entry = (regex, alternate, correct) alternateREs.append(entry) diff --git a/usr/src/tools/onbld/Checks/py2/Makefile b/usr/src/tools/onbld/Checks/py2/Makefile new file mode 100644 index 0000000000..40b2b41255 --- /dev/null +++ b/usr/src/tools/onbld/Checks/py2/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON_VERSION) +PYPKGVERS = $(PYTHON_PKGVERS) +PYOBJS = $(PYSRCS:%.py=%.pyc) + +include ../../../Makefile.python + +all: $(PYVERSOJBS) + +install: all $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber diff --git a/usr/src/tools/onbld/Checks/py3/Makefile b/usr/src/tools/onbld/Checks/py3/Makefile new file mode 100644 index 0000000000..0bf31188b8 --- /dev/null +++ b/usr/src/tools/onbld/Checks/py3/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON3_VERSION) +PYPKGVERS = $(PYTHON3_PKGVERS) +PYOBJS = $(PYSRCS:%.py=__pycache__/%.cpython$(PYTHON3_PKGVERS).pyc) + +include ../../../Makefile.python + +all: +install: $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber + diff --git a/usr/src/tools/onbld/Makefile b/usr/src/tools/onbld/Makefile index 8e5bb7d98f..2127334ae7 100644 --- a/usr/src/tools/onbld/Makefile +++ b/usr/src/tools/onbld/Makefile @@ -23,38 +23,24 @@ # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. # -include $(SRC)/Makefile.master -include ../Makefile.tools - -SUBDIRS= \ - Checks \ - Scm - -PYSRCS = \ - __init__.py - -PYOBJS = $(PYSRCS:%.py=%.pyc) -PYTOPDIR = $(ROOTONBLDLIB) -PYMODDIR = onbld - -all := TARGET = all -install := TARGET = install -clean := TARGET = clean -clobber := TARGET = clobber +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. -include ../Makefile.python - -.KEEP_STATE: - -all: $(PYVERSOBJS) $(SUBDIRS) +include ../Makefile.tools -install: all $(ROOTPYFILES) $(SUBDIRS) +SUBDIRS=Checks Scm +$(BUILDPY2TOOLS)SUBDIRS += py2 +$(BUILDPY3TOOLS)SUBDIRS += py3 -clean: $(SUBDIRS) +all := TARGET= all +install := TARGET= install +clean := TARGET= clean +clobber := TARGET= clobber +lint := TARGET= lint -clobber: clean pyclobber $(SUBDIRS) +all install clean clobber lint: $(SUBDIRS) $(SUBDIRS): FRC @cd $@; pwd; $(MAKE) $(TARGET) FRC: + diff --git a/usr/src/tools/onbld/Makefile.com b/usr/src/tools/onbld/Makefile.com new file mode 100644 index 0000000000..8ccc9a194b --- /dev/null +++ b/usr/src/tools/onbld/Makefile.com @@ -0,0 +1,36 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2014, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. +# + +include $(SRC)/Makefile.master +include ../../Makefile.tools + +PYTOPDIR = $(ROOTONBLDLIB) +PYMODDIR = onbld + +PYSRCS = \ + __init__.py + diff --git a/usr/src/tools/onbld/Scm/Makefile b/usr/src/tools/onbld/Scm/Makefile index c3fbca605f..b6f18bdf85 100644 --- a/usr/src/tools/onbld/Scm/Makefile +++ b/usr/src/tools/onbld/Scm/Makefile @@ -21,26 +21,26 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. -# Copyright (c) 2014, Joyent, Inc. # -include $(SRC)/Makefile.master -include ../../Makefile.tools +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. -PYSRCS = \ - __init__.py \ - Ignore.py +include ../../Makefile.tools -PYOBJS = $(PYSRCS:%.py=%.pyc) -PYTOPDIR = $(ROOTONBLDLIB) -PYMODDIR = onbld/Scm +SUBDIRS= +$(BUILDPY2TOOLS)SUBDIRS += py2 +$(BUILDPY3TOOLS)SUBDIRS += py3 -include ../../Makefile.python +all := TARGET= all +install := TARGET= install +clean := TARGET= clean +clobber := TARGET= clobber +lint := TARGET= lint -all: $(PYVERSOBJS) +all install clean clobber lint: $(SUBDIRS) -install: all $(ROOTPYFILES) +$(SUBDIRS): FRC + @cd $@; pwd; $(MAKE) $(TARGET) -clean: +FRC: -clobber: clean pyclobber diff --git a/usr/src/tools/onbld/Scm/Makefile.com b/usr/src/tools/onbld/Scm/Makefile.com new file mode 100644 index 0000000000..2caeb4e869 --- /dev/null +++ b/usr/src/tools/onbld/Scm/Makefile.com @@ -0,0 +1,37 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2014, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. +# + +include $(SRC)/Makefile.master +include ../../../Makefile.tools + +PYTOPDIR = $(ROOTONBLDLIB) +PYMODDIR = onbld/Scm + +PYSRCS = \ + __init__.py \ + Ignore.py + diff --git a/usr/src/tools/onbld/Scm/py2/Makefile b/usr/src/tools/onbld/Scm/py2/Makefile new file mode 100644 index 0000000000..40b2b41255 --- /dev/null +++ b/usr/src/tools/onbld/Scm/py2/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON_VERSION) +PYPKGVERS = $(PYTHON_PKGVERS) +PYOBJS = $(PYSRCS:%.py=%.pyc) + +include ../../../Makefile.python + +all: $(PYVERSOJBS) + +install: all $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber diff --git a/usr/src/tools/onbld/Scm/py3/Makefile b/usr/src/tools/onbld/Scm/py3/Makefile new file mode 100644 index 0000000000..0bf31188b8 --- /dev/null +++ b/usr/src/tools/onbld/Scm/py3/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON3_VERSION) +PYPKGVERS = $(PYTHON3_PKGVERS) +PYOBJS = $(PYSRCS:%.py=__pycache__/%.cpython$(PYTHON3_PKGVERS).pyc) + +include ../../../Makefile.python + +all: +install: $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber + diff --git a/usr/src/tools/onbld/py2/Makefile b/usr/src/tools/onbld/py2/Makefile new file mode 100644 index 0000000000..5b372bc0ce --- /dev/null +++ b/usr/src/tools/onbld/py2/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON_VERSION) +PYPKGVERS = $(PYTHON_PKGVERS) +PYOBJS = $(PYSRCS:%.py=%.pyc) + +include ../../Makefile.python + +all: $(PYVERSOJBS) + +install: all $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber diff --git a/usr/src/tools/onbld/py3/Makefile b/usr/src/tools/onbld/py3/Makefile new file mode 100644 index 0000000000..dd7c6a78d4 --- /dev/null +++ b/usr/src/tools/onbld/py3/Makefile @@ -0,0 +1,45 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# + +# Copyright 2010, Richard Lowe +# Copyright 2014 Garrett D'Amore <garrett@damore.org> +# Copyright 2016, Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + +include ../Makefile.com + +PYVER = $(PYTHON3_VERSION) +PYPKGVERS = $(PYTHON3_PKGVERS) +PYOBJS = $(PYSRCS:%.py=__pycache__/%.cpython$(PYTHON3_PKGVERS).pyc) + +include ../../Makefile.python + +all: +install: $(ROOTPYFILES) + +clean: + +clobber: clean pyclobber + diff --git a/usr/src/tools/scripts/cddlchk.py b/usr/src/tools/scripts/cddlchk.py index d0b3a70474..793f077bea 100644 --- a/usr/src/tools/scripts/cddlchk.py +++ b/usr/src/tools/scripts/cddlchk.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -22,13 +22,14 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # # Check for valid CDDL blocks in source files. # -import sys, os, getopt, fnmatch +import sys, os, io, getopt, fnmatch sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", "lib", "python%d.%d" % sys.version_info[:2])) @@ -48,7 +49,7 @@ class ExceptionList(object): fh = None try: fh = open(exfile, 'r') - except IOError, e: + except IOError as e: sys.stderr.write('Failed to open exception list: ' '%s: %s\n' % (e.filename, e.strerror)) sys.exit(2) @@ -91,15 +92,15 @@ def usage(): def check(filename, opts): try: - fh = open(filename, 'r') - except IOError, e: + with io.open(filename, encoding='utf-8', + errors='replace') as fh: + return cddlchk(fh, verbose=opts['verbose'], + lenient=opts['lenient'], + output=sys.stdout) + except IOError as e: sys.stderr.write("failed to open '%s': %s\n" % (e.filename, e.strerror)) return 1 - else: - return cddlchk(fh, verbose=opts['verbose'], - lenient=opts['lenient'], - output=sys.stdout) def walker(opts, dirname, fnames): for f in fnames: diff --git a/usr/src/tools/scripts/copyrightchk.py b/usr/src/tools/scripts/copyrightchk.py index 8e6228f5f7..fab81fc7ce 100644 --- a/usr/src/tools/scripts/copyrightchk.py +++ b/usr/src/tools/scripts/copyrightchk.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -22,13 +22,14 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # # Check for valid SMI copyright notices in source files. # -import sys, os +import sys, os, io sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", "lib", "python%d.%d" % sys.version_info[:2])) @@ -41,13 +42,12 @@ from onbld.Checks.Copyright import copyright ret = 0 for filename in sys.argv[1:]: try: - fin = open(filename, 'r') - except IOError, e: + with io.open(filename, encoding='utf-8', + errors='replace') as fin: + ret |= copyright(fin, output=sys.stdout) + except IOError as e: sys.stderr.write("failed to open '%s': %s\n" % (e.filename, e.strerror)) continue - ret |= copyright(fin, output=sys.stdout) - fin.close() - sys.exit(ret) diff --git a/usr/src/tools/scripts/git-pbchk.py b/usr/src/tools/scripts/git-pbchk.py index 7f3956ac42..e0f3d662d1 100644 --- a/usr/src/tools/scripts/git-pbchk.py +++ b/usr/src/tools/scripts/git-pbchk.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 @@ -21,16 +21,23 @@ # Copyright (c) 2015, 2016 by Delphix. All rights reserved. # Copyright 2016 Nexenta Systems, Inc. # Copyright 2018 Joyent, Inc. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # +from __future__ import print_function + import getopt +import io import os import re import subprocess import sys import tempfile -from cStringIO import StringIO +if sys.version_info[0] < 3: + from cStringIO import StringIO +else: + from io import StringIO # # Adjust the load path based on our location and the version of python into @@ -51,7 +58,6 @@ from onbld.Scm import Ignore from onbld.Checks import Comments, Copyright, CStyle, HdrChk, WsCheck from onbld.Checks import JStyle, Keywords, ManLint, Mapfile, SpellCheck - class GitError(Exception): pass @@ -65,15 +71,15 @@ def git(command): command = ["git"] + command try: - tmpfile = tempfile.TemporaryFile(prefix="git-nits") - except EnvironmentError, e: + tmpfile = tempfile.TemporaryFile(prefix="git-nits", mode="w+b") + except EnvironmentError as e: raise GitError("Could not create temporary file: %s\n" % e) try: p = subprocess.Popen(command, stdout=tmpfile, stderr=subprocess.PIPE) - except OSError, e: + except OSError as e: raise GitError("could not execute %s: %s\n" % (command, e)) err = p.wait() @@ -81,38 +87,30 @@ def git(command): raise GitError(p.stderr.read()) tmpfile.seek(0) - return tmpfile - + lines = [] + for l in tmpfile: + lines.append(l.decode('utf-8', 'replace')) + return lines def git_root(): """Return the root of the current git workspace""" p = git('rev-parse --git-dir') + dir = p[0] - if not p: - sys.stderr.write("Failed finding git workspace\n") - sys.exit(err) - - return os.path.abspath(os.path.join(p.readlines()[0], - os.path.pardir)) - + return os.path.abspath(os.path.join(dir, os.path.pardir)) def git_branch(): """Return the current git branch""" p = git('branch') - if not p: - sys.stderr.write("Failed finding git branch\n") - sys.exit(err) - for elt in p: if elt[0] == '*': if elt.endswith('(no branch)'): return None return elt.split()[1] - def git_parent_branch(branch): """Return the parent of the current git branch. @@ -137,7 +135,6 @@ def git_parent_branch(branch): return remote return 'origin/master' - def git_comments(parent): """Return a list of any checkin comments on this git branch""" @@ -147,8 +144,7 @@ def git_comments(parent): sys.stderr.write("Failed getting git comments\n") sys.exit(err) - return [x.strip() for x in p.readlines() if x != ':SEP:\n'] - + return [x.strip() for x in p if x != ':SEP:\n'] def git_file_list(parent, paths=None): """Return the set of files which have ever changed on this branch. @@ -170,17 +166,15 @@ def git_file_list(parent, paths=None): return ret - def not_check(root, cmd): """Return a function which returns True if a file given as an argument should be excluded from the check named by 'cmd'""" - ignorefiles = filter(os.path.exists, + ignorefiles = list(filter(os.path.exists, [os.path.join(root, ".git", "%s.NOT" % cmd), - os.path.join(root, "exception_lists", cmd)]) + os.path.join(root, "exception_lists", cmd)])) return Ignore.ignore(root, ignorefiles) - def gen_files(root, parent, paths, exclude): """Return a function producing file names, relative to the current directory, of any file changed on this branch (limited to 'paths' if @@ -202,20 +196,19 @@ def gen_files(root, parent, paths, exclude): path = relpath(abspath, '.') try: res = git("diff %s HEAD %s" % (parent, path)) - except GitError, e: + except GitError as e: # This ignores all the errors that can be thrown. Usually, this # means that git returned non-zero because the file doesn't # exist, but it could also fail if git can't create a new file # or it can't be executed. Such errors are 1) unlikely, and 2) # will be caught by other invocations of git(). continue - empty = not res.readline() + empty = not res if (os.path.isfile(path) and not empty and select(path) and not exclude(abspath)): yield path return ret - def comchk(root, parent, flist, output): output.write("Comments:\n") @@ -243,81 +236,68 @@ def mapfilechk(root, parent, flist, output): for f in flist(lambda x: MapfileRE.match(x) and not NotMapSuffixRE.match(x)): - fh = open(f, 'r') - ret |= Mapfile.mapfilechk(fh, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= Mapfile.mapfilechk(fh, output=output) return ret - def copyright(root, parent, flist, output): ret = 0 output.write("Copyrights:\n") for f in flist(): - fh = open(f, 'r') - ret |= Copyright.copyright(fh, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= Copyright.copyright(fh, output=output) return ret - def hdrchk(root, parent, flist, output): ret = 0 output.write("Header format:\n") for f in flist(lambda x: x.endswith('.h')): - fh = open(f, 'r') - ret |= HdrChk.hdrchk(fh, lenient=True, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= HdrChk.hdrchk(fh, lenient=True, output=output) return ret - def cstyle(root, parent, flist, output): ret = 0 output.write("C style:\n") for f in flist(lambda x: x.endswith('.c') or x.endswith('.h')): - fh = open(f, 'r') - ret |= CStyle.cstyle(fh, output=output, picky=True, + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= CStyle.cstyle(fh, output=output, picky=True, check_posix_types=True, check_continuation=True) - fh.close() return ret - def jstyle(root, parent, flist, output): ret = 0 output.write("Java style:\n") for f in flist(lambda x: x.endswith('.java')): - fh = open(f, 'r') - ret |= JStyle.jstyle(fh, output=output, picky=True) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= JStyle.jstyle(fh, output=output, picky=True) return ret - def manlint(root, parent, flist, output): ret = 0 output.write("Man page format/spelling:\n") ManfileRE = re.compile(r'.*\.[0-9][a-z]*$', re.IGNORECASE) for f in flist(lambda x: ManfileRE.match(x)): - fh = open(f, 'r') - ret |= ManLint.manlint(fh, output=output, picky=True) - ret |= SpellCheck.spellcheck(fh, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= ManLint.manlint(fh, output=output, picky=True) + ret |= SpellCheck.spellcheck(fh, output=output) return ret def keywords(root, parent, flist, output): ret = 0 output.write("SCCS Keywords:\n") for f in flist(): - fh = open(f, 'r') - ret |= Keywords.keywords(fh, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= Keywords.keywords(fh, output=output) return ret def wscheck(root, parent, flist, output): ret = 0 output.write("white space nits:\n") for f in flist(): - fh = open(f, 'r') - ret |= WsCheck.wscheck(fh, output=output) - fh.close() + with io.open(f, encoding='utf-8', errors='replace') as fh: + ret |= WsCheck.wscheck(fh, output=output) return ret def run_checks(root, parent, cmds, paths='', opts={}): @@ -334,17 +314,16 @@ def run_checks(root, parent, cmds, paths='', opts={}): for cmd in cmds: s = StringIO() - exclude = not_check(root, cmd.func_name) + exclude = not_check(root, cmd.__name__) result = cmd(root, parent, gen_files(root, parent, paths, exclude), output=s) ret |= result if result != 0: - print s.getvalue() + print(s.getvalue()) return ret - def nits(root, parent, paths): cmds = [copyright, cstyle, @@ -353,10 +332,9 @@ def nits(root, parent, paths): keywords, manlint, mapfilechk, - wscheck] + wscheck] run_checks(root, parent, cmds, paths) - def pbchk(root, parent, paths): cmds = [comchk, copyright, @@ -366,17 +344,16 @@ def pbchk(root, parent, paths): keywords, manlint, mapfilechk, - wscheck] + wscheck] run_checks(root, parent, cmds) - def main(cmd, args): parent_branch = None checkname = None try: opts, args = getopt.getopt(args, 'b:c:p:') - except getopt.GetoptError, e: + except getopt.GetoptError as e: sys.stderr.write(str(e) + '\n') sys.stderr.write("Usage: %s [-c check] [-p branch] [path...]\n" % cmd) sys.exit(1) @@ -410,6 +387,6 @@ def main(cmd, args): if __name__ == '__main__': try: main(os.path.basename(sys.argv[0]), sys.argv[1:]) - except GitError, e: + except GitError as e: sys.stderr.write("failed to run git:\n %s\n" % str(e)) sys.exit(1) diff --git a/usr/src/tools/scripts/hdrchk.py b/usr/src/tools/scripts/hdrchk.py index 8c529b8c3d..8f31899623 100644 --- a/usr/src/tools/scripts/hdrchk.py +++ b/usr/src/tools/scripts/hdrchk.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -22,13 +22,14 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # # Check header files conform to ON standards. # -import sys, os, getopt +import sys, os, io, getopt sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", "lib", "python%d.%d" % sys.version_info[:2])) @@ -59,11 +60,10 @@ for opt, arg in opts: ret = 0 for filename in args: try: - fh = open(filename, 'r') - except IOError, e: + with io.open(filename, encoding='utf-8', + errors='replace') as fh: + ret |= hdrchk(fh, lenient=lenient, output=sys.stderr) + except IOError as e: sys.stderr.write("failed to open '%s': %s\n" % (e.filename, e.strerror)) - else: - ret |= hdrchk(fh, lenient=lenient, output=sys.stderr) - fh.close() sys.exit(ret) diff --git a/usr/src/tools/scripts/mapfilechk.py b/usr/src/tools/scripts/mapfilechk.py index 0e3ba283a5..7429eba099 100644 --- a/usr/src/tools/scripts/mapfilechk.py +++ b/usr/src/tools/scripts/mapfilechk.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -22,13 +22,14 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # # Check for valid link-editor mapfile comment blocks in source files. # -import sys, os, getopt, fnmatch +import sys, os, io, getopt, fnmatch sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", "lib", "python%d.%d" % sys.version_info[:2])) @@ -48,7 +49,7 @@ class ExceptionList(object): fh = None try: fh = open(exfile, 'r') - except IOError, e: + except IOError as e: sys.stderr.write('Failed to open exception list: ' '%s: %s\n' % (e.filename, e.strerror)) sys.exit(2) @@ -90,14 +91,14 @@ def usage(): def check(filename, opts): try: - fh = open(filename, 'r') - except IOError, e: + with io.open(filename, encoding='utf-8', + errors='replace') as fh: + return mapfilechk(fh, verbose=opts['verbose'], + output=sys.stdout) + except IOError as e: sys.stderr.write("failed to open '%s': %s\n" % (e.filename, e.strerror)) return 1 - else: - return mapfilechk(fh, verbose=opts['verbose'], - output=sys.stdout) def walker(opts, dirname, fnames): for f in fnames: diff --git a/usr/src/tools/scripts/validate_pkg.py b/usr/src/tools/scripts/validate_pkg.py index 6e5858a6ae..77e7d8e5d5 100644 --- a/usr/src/tools/scripts/validate_pkg.py +++ b/usr/src/tools/scripts/validate_pkg.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -25,6 +25,8 @@ # Use is subject to license terms. # +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. + # # Compare the content generated by a build to a set of manifests # describing how that content is to be delivered. @@ -132,13 +134,13 @@ class FileInfo(object): mo = m & stat.S_IRWXO e = self.editable - if (((mu & 02) == 0 and (mo & mg & 04) == 04) or - (t == "file" and mo & 01 == 1) or + if (((mu & 0o2) == 0 and (mo & mg & 0o4) == 0o4) or + (t == "file" and mo & 0o1 == 1) or (mg, mo) == (mu, mu) or ((t == "file" and not e or t == "dir" and o == "bin") and - (mg & 05 == mo & 05)) or - (t == "file" and o == "bin" and mu & 01 == 01) or - (m & 0105 != 0 and p.startswith("etc/security/dev/"))): + (mg & 0o5 == mo & 0o5)) or + (t == "file" and o == "bin" and mu & 0o1 == 0o1) or + (m & 0o105 != 0 and p.startswith("etc/security/dev/"))): w.extend(["%s: owner \"%s\" may be safely " \ "changed to \"root\"" % (p, o)]) @@ -405,38 +407,38 @@ class DirectoryTree(dict): def compare(self, other): """Compare two different sets of FileInfo objects. """ - keys1 = frozenset(self.keys()) - keys2 = frozenset(other.keys()) + keys1 = frozenset(list(self.keys())) + keys2 = frozenset(list(other.keys())) common = keys1.intersection(keys2) onlykeys1 = keys1.difference(common) onlykeys2 = keys2.difference(common) if onlykeys1: - print "Entries present in %s but not %s:" % \ - (self.name, other.name) + print("Entries present in %s but not %s:" % \ + (self.name, other.name)) for path in sorted(onlykeys1): - print("\t%s" % str(self[path])) - print "" + print(("\t%s" % str(self[path]))) + print("") if onlykeys2: - print "Entries present in %s but not %s:" % \ - (other.name, self.name) + print("Entries present in %s but not %s:" % \ + (other.name, self.name)) for path in sorted(onlykeys2): - print("\t%s" % str(other[path])) - print "" + print(("\t%s" % str(other[path]))) + print("") nodifferences = True for path in sorted(common): if self[path] != other[path]: if nodifferences: nodifferences = False - print "Entries that differ between %s and %s:" \ - % (self.name, other.name) - print("%14s %s" % (self.name, self[path])) - print("%14s %s" % (other.name, other[path])) + print("Entries that differ between %s and %s:" \ + % (self.name, other.name)) + print(("%14s %s" % (self.name, self[path]))) + print(("%14s %s" % (other.name, other[path]))) if not nodifferences: - print "" + print("") class BadProtolistFormat(Exception): @@ -468,7 +470,7 @@ class ProtoTree(DirectoryTree): if path not in exceptions: try: newentries[path] = RealFileInfo(pdir, path) - except OSError, e: + except OSError as e: sys.stderr.write("Warning: unable to stat %s: %s\n" % (path, e)) continue @@ -491,10 +493,10 @@ class ProtoTree(DirectoryTree): # that cross proto dir boundaries. # hk2path = {} - for path, fileinfo in newentries.iteritems(): + for path, fileinfo in newentries.items(): if fileinfo.hardkey: hk2path.setdefault(fileinfo.hardkey, set()).add(path) - for fileinfo in newentries.itervalues(): + for fileinfo in newentries.values(): if fileinfo.hardkey: fileinfo.hardpaths.update(hk2path[fileinfo.hardkey]) self.update(newentries) @@ -510,7 +512,7 @@ class ProtoTree(DirectoryTree): try: plist = open(protolist) - except IOError, exc: + except IOError as exc: raise IOError("cannot open proto list: %s" % str(exc)) newentries = {} @@ -543,10 +545,10 @@ class ProtoTree(DirectoryTree): plist.close() hk2path = {} - for path, fileinfo in newentries.iteritems(): + for path, fileinfo in newentries.items(): if fileinfo.hardkey: hk2path.setdefault(fileinfo.hardkey, set()).add(path) - for fileinfo in newentries.itervalues(): + for fileinfo in newentries.values(): if fileinfo.hardkey: fileinfo.hardpaths.update(hk2path[fileinfo.hardkey]) self.update(newentries) @@ -582,9 +584,9 @@ class ManifestTree(DirectoryTree): mfest = manifest.Manifest() try: mfest.set_content(open(os.path.join(root, mfile)).read()) - except IOError, exc: + except IOError as exc: raise IOError("cannot read manifest: %s" % str(exc)) - except actions.ActionError, exc: + except actions.ActionError as exc: raise ManifestParsingError(mfile, str(exc)) # @@ -657,10 +659,10 @@ class ManifestTree(DirectoryTree): modewarnings.update(self[path].checkmodes(modechecks)) if len(modewarnings) > 0: - print "warning: unsafe permissions in %s" % mfile + print("warning: unsafe permissions in %s" % mfile) for w in sorted(modewarnings): - print w - print "" + print(w) + print("") def adddir(self, mdir, arch, modechecks, exceptions): """Walks the specified directory looking for pkg(5) manifests. @@ -670,14 +672,14 @@ class ManifestTree(DirectoryTree): stat.S_ISREG(os.lstat(os.path.join(mdir, mfile)).st_mode)): try: self.addmanifest(mdir, mfile, arch, modechecks, exceptions) - except IOError, exc: + except IOError as exc: sys.stderr.write("warning: %s\n" % str(exc)) def resolvehardlinks(self): """Populates mode, group, and owner for resolved (ie link target is present in the manifest tree) hard links. """ - for info in self.values(): + for info in list(self.values()): if info.name() == "hardlink": tgt = info.hardkey if tgt in self: @@ -696,7 +698,7 @@ class ExceptionList(set): for fname in files: try: self.readexceptionfile(fname, arch) - except IOError, exc: + except IOError as exc: sys.stderr.write("warning: cannot read exception file: %s\n" % str(exc)) @@ -772,7 +774,7 @@ def main(argv): """ try: opts, args = getopt.getopt(argv, 'a:e:Ll:Mm:p:vX:') - except getopt.GetoptError, exc: + except getopt.GetoptError as exc: usage(str(exc)) if args: @@ -849,8 +851,8 @@ def main(argv): manifesttree.adddir(mdir, arch, modechecks, exceptions) if listonly: manifesttree.resolvehardlinks() - for info in manifesttree.values(): - print "%s" % info.protostr() + for info in list(manifesttree.values()): + print("%s" % info.protostr()) sys.exit(0) if modechecks is not None: sys.exit(0) @@ -865,24 +867,24 @@ def main(argv): for plist in protolists: try: protolist.addprotolist(plist, exceptions) - except IOError, exc: + except IOError as exc: sys.stderr.write("warning: %s\n" % str(exc)) trees.append(protolist) if verbose and exceptions: - print "Entries present in exception list but missing from proto area:" + print("Entries present in exception list but missing from proto area:") for exc in sorted(exceptions): - print "\t%s" % exc - print "" + print("\t%s" % exc) + print("") usedexceptions = originalexceptions.difference(exceptions) harmfulexceptions = usedexceptions.intersection(manifesttree) if harmfulexceptions: - print "Entries present in exception list but also in manifests:" + print("Entries present in exception list but also in manifests:") for exc in sorted(harmfulexceptions): - print "\t%s" % exc + print("\t%s" % exc) del manifesttree[exc] - print "" + print("") trees[0].compare(trees[1]) diff --git a/usr/src/tools/scripts/wscheck.py b/usr/src/tools/scripts/wscheck.py index ce85b8971f..ee3fd83b1d 100644 --- a/usr/src/tools/scripts/wscheck.py +++ b/usr/src/tools/scripts/wscheck.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -22,6 +22,7 @@ # # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -29,7 +30,7 @@ # (space tab, trailing space) # -import sys, os +import sys, os, io sys.path.insert(1, os.path.join(os.path.dirname(__file__), "..", "lib", "python%d.%d" % sys.version_info[:2])) @@ -42,13 +43,12 @@ from onbld.Checks.WsCheck import wscheck ret = 0 for filename in sys.argv[1:]: try: - fh = open(filename, 'r') - except IOError, e: + with io.open(filename, encoding='utf-8', + errors='replace') as fh: + ret |= wscheck(fh, output=sys.stderr) + except IOError as e: sys.stderr.write("failed to open '%s': %s\n" % (e.filename, e.strerror)) - continue - - ret |= wscheck(fh, output=sys.stderr) - fh.close() + continue sys.exit(ret) diff --git a/usr/src/tools/scripts/wsdiff.py b/usr/src/tools/scripts/wsdiff.py index 367903a190..86b805b30f 100644 --- a/usr/src/tools/scripts/wsdiff.py +++ b/usr/src/tools/scripts/wsdiff.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -20,6 +20,7 @@ # CDDL HEADER END # # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -69,7 +70,8 @@ # -r Log results and observed differences # -i Tell wsdiff which objects to compare via an input file list -import datetime, fnmatch, getopt, os, profile, commands +from __future__ import print_function +import datetime, fnmatch, getopt, os, profile, subprocess import re, resource, select, shutil, signal, string, struct, sys, tempfile import time, threading from stat import * @@ -105,11 +107,12 @@ wsdiff_path = [ "/usr/bin", # Ideally, we would purge use the use of __DATE__ from the source, but because # this is source we wish to distribute with Solaris "unchanged", we cannot modify. # -wsdiff_exceptions = [ "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1" - ] +wsdiff_exceptions = [ + "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1" +] ##### # Logging routines @@ -122,12 +125,12 @@ def debug(msg) : msg = "## " + msg if debugon : output_lock.acquire() - print >> sys.stdout, msg + print(msg) sys.stdout.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() @@ -135,25 +138,25 @@ def debug(msg) : def info(msg) : output_lock.acquire() - print >> sys.stdout, msg + print(msg) sys.stdout.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() # Error message to be printed to the screen, and the log file def error(msg) : - + output_lock.acquire() - print >> sys.stderr, "ERROR:", msg + print("ERROR: " + msg, file=sys.stderr) sys.stderr.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, "ERROR:", msg + print("ERROR: " + msg, file=log) log.flush() log_lock.release() @@ -162,10 +165,10 @@ def v_info(msg) : if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() - + # # Flag a detected file difference # Display the fileName to stdout, and log the difference @@ -179,7 +182,7 @@ def difference(f, dtype, diffs) : if sorted : differentFiles.append(f) else: - print >> sys.stdout, f + print(f) sys.stdout.flush() output_lock.release() @@ -192,21 +195,20 @@ def log_difference(f, dtype, diffs) : if logging : log_lock.acquire() - print >> log, f - print >> log, "NOTE:", dtype, "difference detected." + print(f, file=log) + print("NOTE: " + dtype + " difference detected.", file=log) difflen = len(diffs) if difflen > 0 : - print >> log + print('', file=log) if not vdiffs and difflen > diffs_sz_thresh : - print >> log, diffs[:diffs_sz_thresh] - print >> log, \ - "... truncated due to length: " \ - "use -v to override ..." + print(diffs[:diffs_sz_thresh], file=log) + print("... truncated due to length: " + + "use -v to override ...", file=log) else : - print >> log, diffs - print >> log, "\n" + print(diffs, file=log) + print('\n', file=log) log.flush() log_lock.release() @@ -228,29 +230,30 @@ def diffFileData(tmpf1, tmpf2) : binaries = True tmp_od1 = tmpf1 + ".od" tmp_od2 = tmpf2 + ".od" - + cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1 os.system(cmd) cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2 os.system(cmd) - + tmpf1 = tmp_od1 tmpf2 = tmp_od2 try: - data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2) + data = subprocess.check_output( + diff_cmd + " " + tmpf1 + " " + tmpf2) # Remove the temp files as we no longer need them. if binaries : try: os.unlink(tmp_od1) - except OSError, e: - error("diffFileData: unlink failed %s" % e) + except OSError as e: + error("diffFileData: unlink failed %s" % e) try: os.unlink(tmp_od2) - except OSError, e: - error("diffFileData: unlink failed %s" % e) + except OSError as e: + error("diffFileData: unlink failed %s" % e) except: - error("failed to get output of command: " + diff_cmd + " " \ + error("failed to get output of command: " + diff_cmd + " " + tmpf1 + " " + tmpf2) # Send exception for the failed command up @@ -323,14 +326,15 @@ def fnFormat(fn) : # def usage() : sys.stdout.flush() - print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new + print("""Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new -d Print debug messages about the progress -v Do not truncate observed diffs in results -V Log *all* ELF sect diffs vs. logging the first diff found -t Use onbld tools in $SRC/tools -r Log results and observed differences -s Produce sorted list of differences - -i Tell wsdiff which objects to compare via an input file list""" + -i Tell wsdiff which objects to compare via an input file list""", + file=sys.stderr) sys.exit(1) # @@ -360,7 +364,7 @@ def args() : # arguments, the paths to the base (old), ptch (new) proto areas try: optlist, args = getopt.getopt(sys.argv[1:], validOpts) - except getopt.error, val: + except getopt.error as val: usage() if len(args) != 2 : @@ -508,7 +512,7 @@ def protoCatalog(base, ptch) : baseFilesList = list(findFiles(base)) baseStringLength = len(base) debug("Found " + str(len(baseFilesList)) + " files") - + debug("Getting the list of files in the patch area"); ptchFilesList = list(findFiles(ptch)) ptchStringLength = len(ptch) @@ -546,10 +550,10 @@ def protoCatalog(base, ptch) : debug("Eliminating deleted files from the list of objects") for fileName in deletedFiles : try: - compFiles.remove(fileName) + compFiles.remove(fileName) except: error("filelist.remove() failed") - debug("List for comparison reduced to " + str(len(compFiles)) \ + debug("List for comparison reduced to " + str(len(compFiles)) + " files") # New files appear in the patch area, but not the base @@ -628,10 +632,10 @@ def flistCatalog(base, ptch, flist) : elif ptch_present : newFiles.append(fn) else : - if os.path.islink(base + fn) and \ - os.path.islink(ptch + fn) : + if (os.path.islink(base + fn) and + os.path.islink(ptch + fn)) : continue - error(f + " in file list, but not in either tree. " + \ + error(f + " in file list, but not in either tree. " + "Skipping...") return compFiles, newFiles, deletedFiles @@ -674,7 +678,7 @@ def get_elfheader(f) : header = {} - hstring = commands.getoutput(elfdump_cmd + " -c " + f) + hstring = subprocess.check_output(elfdump_cmd + " -c " + f) if len(hstring) == 0 : error("Failed to dump ELF header for " + f) @@ -691,7 +695,7 @@ def get_elfheader(f) : section = sect[datap:].split()[1] datap = sect.find("sh_type:"); if datap == -1 : - error("Could not get type for sect: " + section + \ + error("Could not get type for sect: " + section + " in " + f) sh_type = sect[datap:].split()[2] header[section] = sh_type @@ -703,10 +707,10 @@ def get_elfheader(f) : # def extract_elf_section(f, section) : - data = commands.getoutput(dump_cmd + " -sn " + section + " " + f) + data = subprocess.check_output(dump_cmd + " -sn " + section + " " + f) if len(data) == 0 : - error(dump_cmd + "yielded no data on section " + section + \ + error(dump_cmd + "yielded no data on section " + section + " of " + f) raise return @@ -754,24 +758,24 @@ def diff_elf_section(f1, f2, section, sh_type) : cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2 elif (section == ".symtab" or section == ".dynsym") : - cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \ - " > " + tmpFile1 - cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \ - " > " + tmpFile2 + cmd1 = (elfdump_cmd + " -s -N " + section + " " + f1 + + " > " + tmpFile1) + cmd2 = (elfdump_cmd + " -s -N " + section + " " + f2 + + " > " + tmpFile2) elif (section in text_sections) : # dis sometimes complains when it hits something it doesn't # know how to disassemble. Just ignore it, as the output # being generated here is human readable, and we've already # correctly flagged the difference. - cmd1 = dis_cmd + " -t " + section + " " + f1 + \ - " 2>/dev/null | grep -v disassembly > " + tmpFile1 - cmd2 = dis_cmd + " -t " + section + " " + f2 + \ - " 2>/dev/null | grep -v disassembly > " + tmpFile2 + cmd1 = (dis_cmd + " -t " + section + " " + f1 + + " 2>/dev/null | grep -v disassembly > " + tmpFile1) + cmd2 = (dis_cmd + " -t " + section + " " + f2 + + " 2>/dev/null | grep -v disassembly > " + tmpFile2) else : - cmd1 = elfdump_cmd + " -w " + tmpFile1 + " -N " + \ - section + " " + f1 - cmd2 = elfdump_cmd + " -w " + tmpFile2 + " -N " + \ - section + " " + f2 + cmd1 = (elfdump_cmd + " -w " + tmpFile1 + " -N " + + section + " " + f1) + cmd2 = (elfdump_cmd + " -w " + tmpFile2 + " -N " + + section + " " + f2) os.system(cmd1) os.system(cmd2) @@ -781,12 +785,12 @@ def diff_elf_section(f1, f2, section, sh_type) : # remove temp files as we no longer need them try: os.unlink(tmpFile1) - except OSError, e: - error("diff_elf_section: unlink failed %s" % e) + except OSError as e: + error("diff_elf_section: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("diff_elf_section: unlink failed %s" % e) + except OSError as e: + error("diff_elf_section: unlink failed %s" % e) return (data) @@ -830,7 +834,7 @@ def compareElfs(base, ptch, quiet) : base_header = get_elfheader(base) except: return - sections = base_header.keys() + sections = base_header.keys() try: ptch_header = get_elfheader(ptch) @@ -859,25 +863,25 @@ def compareElfs(base, ptch, quiet) : slist = "" for sect in e1_only_sections : slist = slist + sect + "\t" - data = "ELF sections found in " + \ - base + " but not in " + ptch + \ - "\n\n" + slist + data = ("ELF sections found in " + + base + " but not in " + ptch + + "\n\n" + slist) difference(fileName, "ELF", data) return 1 - + if len(e2_only_sections) > 0 : if quiet : return 1 - + data = "" if logging : slist = "" for sect in e2_only_sections : slist = slist + sect + "\t" - data = "ELF sections found in " + \ - ptch + " but not in " + base + \ - "\n\n" + slist + data = ("ELF sections found in " + + ptch + " but not in " + base + + "\n\n" + slist) difference(fileName, "ELF", data) return 1 @@ -909,7 +913,7 @@ def compareElfs(base, ptch, quiet) : if len(s1) != len (s2) or s1 != s2: if not quiet: sh_type = base_header[sect] - data = diff_elf_section(base, ptch, \ + data = diff_elf_section(base, ptch, sect, sh_type) # If all ELF sections are being reported, then @@ -917,10 +921,10 @@ def compareElfs(base, ptch, quiet) : # stdout only once. Any other section differences # should be logged to the results file directly if not first_section : - log_difference(fileName, \ + log_difference(fileName, "ELF " + sect, data) else : - difference(fileName, "ELF " + sect, \ + difference(fileName, "ELF " + sect, data) if not reportAllSects : @@ -954,7 +958,7 @@ def compareArchives(base, ptch, fileType) : fileName = fnFormat(base) t = threading.currentThread() - ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() + ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName() # @@ -966,42 +970,44 @@ def compareArchives(base, ptch, fileType) : try: os.makedirs(ArchTmpDir1) - except OSError, e: - error("compareArchives: makedir failed %s" % e) + except OSError as e: + error("compareArchives: makedir failed %s" % e) return -1 try: os.makedirs(ArchTmpDir2) - except OSError, e: - error("compareArchives: makedir failed %s" % e) + except OSError as e: + error("compareArchives: makedir failed %s" % e) return -1 # copy over the objects to the temp areas, and # unpack them baseCmd = "cp -fp " + base + " " + ArchTmpDir1 - status, output = commands.getstatusoutput(baseCmd) - if status != 0 : + try: + output = subprocess.check_output(baseCmd) + except CalledProcessError: error(baseCmd + " failed: " + output) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2 - status, output = commands.getstatusoutput(ptchCmd) - if status != 0 : + try: + output = subprocess.check_output(ptchCmd) + except CalledProcessError: error(ptchCmd + " failed: " + output) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 bname = string.split(fileName, '/')[-1] if fileType == "Java Archive" : - baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \ - "; rm -f " + bname + " META-INF/MANIFEST.MF" - ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \ - "; rm -f " + bname + " META-INF/MANIFEST.MF" + baseCmd = ("cd " + ArchTmpDir1 + "; " + "jar xf " + bname + + "; rm -f " + bname + " META-INF/MANIFEST.MF") + ptchCmd = ("cd " + ArchTmpDir2 + "; " + "jar xf " + bname + + "; rm -f " + bname + " META-INF/MANIFEST.MF") elif fileType == "ELF Object Archive" : - baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \ - bname + "; rm -f " + bname - ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \ - bname + "; rm -f " + bname + baseCmd = ("cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + + bname + "; rm -f " + bname) + ptchCmd = ("cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + + bname + "; rm -f " + bname) else : error("unexpected file type: " + fileType) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) @@ -1026,22 +1032,22 @@ def compareArchives(base, ptch, fileType) : for fn in ptchFlist : if not fn in baseFlist : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " added to " + fileName) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 for fn in baseFlist : if not fn in ptchFlist : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " removed from " + fileName) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 - differs = compareOneFile((ArchTmpDir1 + fn), \ + differs = compareOneFile((ArchTmpDir1 + fn), (ArchTmpDir2 + fn), True) if differs : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " in " + fileName + " differs") clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 @@ -1091,13 +1097,13 @@ def compareBasic(base, ptch, quiet, fileType) : if needToSnip : toSnipBegin = string.find(baseData, toSnipBeginStr) if toSnipBegin != -1 : - toSnipEnd = string.find(baseData[toSnipBegin:], \ - toSnipEndStr) + \ - len(toSnipEndStr) - baseData = baseData[:toSnipBegin] + \ - baseData[toSnipBegin + toSnipEnd:] - ptchData = ptchData[:toSnipBegin] + \ - ptchData[toSnipBegin + toSnipEnd:] + toSnipEnd = (string.find(baseData[toSnipBegin:], + toSnipEndStr) + + len(toSnipEndStr)) + baseData = (baseData[:toSnipBegin] + + baseData[toSnipBegin + toSnipEnd:]) + ptchData = (ptchData[:toSnipBegin] + + ptchData[toSnipBegin + toSnipEnd:]) if quiet : if baseData != ptchData : @@ -1126,20 +1132,20 @@ def compareByDumping(base, ptch, quiet, fileType) : tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() if fileType == "Lint Library" : - baseCmd = lintdump_cmd + " -ir " + base + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + \ - " | grep -v PASS[1-3]:" + \ - " > " + tmpFile1 - ptchCmd = lintdump_cmd + " -ir " + ptch + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + \ - " | grep -v PASS[1-3]:" + \ - " > " + tmpFile2 + baseCmd = (lintdump_cmd + " -ir " + base + + " | egrep -v '(LINTOBJ|LINTMOD):'" + + " | grep -v PASS[1-3]:" + + " > " + tmpFile1) + ptchCmd = (lintdump_cmd + " -ir " + ptch + + " | egrep -v '(LINTOBJ|LINTMOD):'" + + " | grep -v PASS[1-3]:" + + " > " + tmpFile2) elif fileType == "Sqlite Database" : - baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \ - tmpFile1 - ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \ - tmpFile2 - + baseCmd = ("echo .dump | " + sqlite_cmd + base + " > " + + tmpFile1) + ptchCmd = ("echo .dump | " + sqlite_cmd + ptch + " > " + + tmpFile2) + os.system(baseCmd) os.system(ptchCmd) @@ -1165,24 +1171,24 @@ def compareByDumping(base, ptch, quiet, fileType) : data = diffFileData(tmpFile1, tmpFile2); try: os.unlink(tmpFile1) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) difference(fileName, fileType, data) - return 1 + return 1 # Remove the temporary files now. try: os.unlink(tmpFile1) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) return 0 @@ -1194,7 +1200,7 @@ def compareByDumping(base, ptch, quiet, fileType) : def discontinue_processing(signl, frme): global keep_processing - print >> sys.stderr, "Caught Ctrl-C, stopping the threads" + print("Caught Ctrl-C, stopping the threads", file=sys.stderr) keep_processing = False return 0 @@ -1204,29 +1210,29 @@ def discontinue_processing(signl, frme): # worker thread for changedFiles processing # class workerThread(threading.Thread) : - def run(self): - global wset_lock - global changedFiles - global baseRoot - global ptchRoot - global keep_processing - - while (keep_processing) : - # grab the lock to changedFiles and remove one member - # and process it - wset_lock.acquire() - try : - fn = changedFiles.pop() - except IndexError : - # there is nothing more to do + def run(self): + global wset_lock + global changedFiles + global baseRoot + global ptchRoot + global keep_processing + + while (keep_processing) : + # grab the lock to changedFiles and remove one member + # and process it + wset_lock.acquire() + try : + fn = changedFiles.pop() + except IndexError : + # there is nothing more to do + wset_lock.release() + return wset_lock.release() - return - wset_lock.release() - base = baseRoot + fn - ptch = ptchRoot + fn + base = baseRoot + fn + ptch = ptchRoot + fn - compareOneFile(base, ptch, False) + compareOneFile(base, ptch, False) ##### @@ -1278,10 +1284,10 @@ def cleanup(ret) : debug("Performing cleanup (" + str(ret) + ")") if os.path.isdir(tmpDir1) > 0 : shutil.rmtree(tmpDir1) - + if os.path.isdir(tmpDir2) > 0 : shutil.rmtree(tmpDir2) - + if logging : log.close() @@ -1352,13 +1358,13 @@ def main() : if sorted : differentFiles = [] - # + # # Build paths to the tools required tools # # Try to look for tools in $SRC/tools if the "-t" option # was specified # - arch = commands.getoutput("uname -p") + arch = subprocess.check_output("uname -p") if localTools : try: src = os.environ['SRC'] @@ -1402,8 +1408,7 @@ def main() : sys.exit(1) if not os.path.exists(ptchRoot) : - error("new proto area: " + ptchRoot + \ - " does not exist") + error("new proto area: " + ptchRoot + " does not exist") sys.exit(1) # @@ -1422,12 +1427,12 @@ def main() : tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/" try: os.makedirs(tmpDir1) - except OSError, e: - error("main: makedir failed %s" % e) + except OSError as e: + error("main: makedir failed %s" % e) try: os.makedirs(tmpDir2) - except OSError, e: - error("main: makedir failed %s" % e) + except OSError as e: + error("main: makedir failed %s" % e) # Derive a catalog of new, deleted, and to-be-compared objects # either from the specified base and patch proto areas, or from @@ -1471,7 +1476,7 @@ def main() : # the appropriate comparison routine, where the compare # will happen, and any differences will be reported / logged - # determine maximum number of worker threads by using + # determine maximum number of worker threads by using # DMAKE_MAX_JOBS environment variable set by nightly(1) # or get number of CPUs in the system try: @@ -1527,4 +1532,3 @@ if __name__ == '__main__' : except KeyboardInterrupt : cleanup(1); - |