diff options
Diffstat (limited to 'usr/src/tools/scripts/wsdiff.py')
| -rw-r--r-- | usr/src/tools/scripts/wsdiff.py | 320 |
1 files changed, 162 insertions, 158 deletions
diff --git a/usr/src/tools/scripts/wsdiff.py b/usr/src/tools/scripts/wsdiff.py index 367903a190..86b805b30f 100644 --- a/usr/src/tools/scripts/wsdiff.py +++ b/usr/src/tools/scripts/wsdiff.py @@ -1,4 +1,4 @@ -#!@PYTHON@ +#!@TOOLS_PYTHON@ # # CDDL HEADER START # @@ -20,6 +20,7 @@ # CDDL HEADER END # # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright 2018 OmniOS Community Edition (OmniOSce) Association. # # @@ -69,7 +70,8 @@ # -r Log results and observed differences # -i Tell wsdiff which objects to compare via an input file list -import datetime, fnmatch, getopt, os, profile, commands +from __future__ import print_function +import datetime, fnmatch, getopt, os, profile, subprocess import re, resource, select, shutil, signal, string, struct, sys, tempfile import time, threading from stat import * @@ -105,11 +107,12 @@ wsdiff_path = [ "/usr/bin", # Ideally, we would purge use the use of __DATE__ from the source, but because # this is source we wish to distribute with Solaris "unchanged", we cannot modify. # -wsdiff_exceptions = [ "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1", - "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1" - ] +wsdiff_exceptions = [ + "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1", + "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1" +] ##### # Logging routines @@ -122,12 +125,12 @@ def debug(msg) : msg = "## " + msg if debugon : output_lock.acquire() - print >> sys.stdout, msg + print(msg) sys.stdout.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() @@ -135,25 +138,25 @@ def debug(msg) : def info(msg) : output_lock.acquire() - print >> sys.stdout, msg + print(msg) sys.stdout.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() # Error message to be printed to the screen, and the log file def error(msg) : - + output_lock.acquire() - print >> sys.stderr, "ERROR:", msg + print("ERROR: " + msg, file=sys.stderr) sys.stderr.flush() output_lock.release() if logging : log_lock.acquire() - print >> log, "ERROR:", msg + print("ERROR: " + msg, file=log) log.flush() log_lock.release() @@ -162,10 +165,10 @@ def v_info(msg) : if logging : log_lock.acquire() - print >> log, msg + print(msg, file=log) log.flush() log_lock.release() - + # # Flag a detected file difference # Display the fileName to stdout, and log the difference @@ -179,7 +182,7 @@ def difference(f, dtype, diffs) : if sorted : differentFiles.append(f) else: - print >> sys.stdout, f + print(f) sys.stdout.flush() output_lock.release() @@ -192,21 +195,20 @@ def log_difference(f, dtype, diffs) : if logging : log_lock.acquire() - print >> log, f - print >> log, "NOTE:", dtype, "difference detected." + print(f, file=log) + print("NOTE: " + dtype + " difference detected.", file=log) difflen = len(diffs) if difflen > 0 : - print >> log + print('', file=log) if not vdiffs and difflen > diffs_sz_thresh : - print >> log, diffs[:diffs_sz_thresh] - print >> log, \ - "... truncated due to length: " \ - "use -v to override ..." + print(diffs[:diffs_sz_thresh], file=log) + print("... truncated due to length: " + + "use -v to override ...", file=log) else : - print >> log, diffs - print >> log, "\n" + print(diffs, file=log) + print('\n', file=log) log.flush() log_lock.release() @@ -228,29 +230,30 @@ def diffFileData(tmpf1, tmpf2) : binaries = True tmp_od1 = tmpf1 + ".od" tmp_od2 = tmpf2 + ".od" - + cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1 os.system(cmd) cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2 os.system(cmd) - + tmpf1 = tmp_od1 tmpf2 = tmp_od2 try: - data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2) + data = subprocess.check_output( + diff_cmd + " " + tmpf1 + " " + tmpf2) # Remove the temp files as we no longer need them. if binaries : try: os.unlink(tmp_od1) - except OSError, e: - error("diffFileData: unlink failed %s" % e) + except OSError as e: + error("diffFileData: unlink failed %s" % e) try: os.unlink(tmp_od2) - except OSError, e: - error("diffFileData: unlink failed %s" % e) + except OSError as e: + error("diffFileData: unlink failed %s" % e) except: - error("failed to get output of command: " + diff_cmd + " " \ + error("failed to get output of command: " + diff_cmd + " " + tmpf1 + " " + tmpf2) # Send exception for the failed command up @@ -323,14 +326,15 @@ def fnFormat(fn) : # def usage() : sys.stdout.flush() - print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new + print("""Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new -d Print debug messages about the progress -v Do not truncate observed diffs in results -V Log *all* ELF sect diffs vs. logging the first diff found -t Use onbld tools in $SRC/tools -r Log results and observed differences -s Produce sorted list of differences - -i Tell wsdiff which objects to compare via an input file list""" + -i Tell wsdiff which objects to compare via an input file list""", + file=sys.stderr) sys.exit(1) # @@ -360,7 +364,7 @@ def args() : # arguments, the paths to the base (old), ptch (new) proto areas try: optlist, args = getopt.getopt(sys.argv[1:], validOpts) - except getopt.error, val: + except getopt.error as val: usage() if len(args) != 2 : @@ -508,7 +512,7 @@ def protoCatalog(base, ptch) : baseFilesList = list(findFiles(base)) baseStringLength = len(base) debug("Found " + str(len(baseFilesList)) + " files") - + debug("Getting the list of files in the patch area"); ptchFilesList = list(findFiles(ptch)) ptchStringLength = len(ptch) @@ -546,10 +550,10 @@ def protoCatalog(base, ptch) : debug("Eliminating deleted files from the list of objects") for fileName in deletedFiles : try: - compFiles.remove(fileName) + compFiles.remove(fileName) except: error("filelist.remove() failed") - debug("List for comparison reduced to " + str(len(compFiles)) \ + debug("List for comparison reduced to " + str(len(compFiles)) + " files") # New files appear in the patch area, but not the base @@ -628,10 +632,10 @@ def flistCatalog(base, ptch, flist) : elif ptch_present : newFiles.append(fn) else : - if os.path.islink(base + fn) and \ - os.path.islink(ptch + fn) : + if (os.path.islink(base + fn) and + os.path.islink(ptch + fn)) : continue - error(f + " in file list, but not in either tree. " + \ + error(f + " in file list, but not in either tree. " + "Skipping...") return compFiles, newFiles, deletedFiles @@ -674,7 +678,7 @@ def get_elfheader(f) : header = {} - hstring = commands.getoutput(elfdump_cmd + " -c " + f) + hstring = subprocess.check_output(elfdump_cmd + " -c " + f) if len(hstring) == 0 : error("Failed to dump ELF header for " + f) @@ -691,7 +695,7 @@ def get_elfheader(f) : section = sect[datap:].split()[1] datap = sect.find("sh_type:"); if datap == -1 : - error("Could not get type for sect: " + section + \ + error("Could not get type for sect: " + section + " in " + f) sh_type = sect[datap:].split()[2] header[section] = sh_type @@ -703,10 +707,10 @@ def get_elfheader(f) : # def extract_elf_section(f, section) : - data = commands.getoutput(dump_cmd + " -sn " + section + " " + f) + data = subprocess.check_output(dump_cmd + " -sn " + section + " " + f) if len(data) == 0 : - error(dump_cmd + "yielded no data on section " + section + \ + error(dump_cmd + "yielded no data on section " + section + " of " + f) raise return @@ -754,24 +758,24 @@ def diff_elf_section(f1, f2, section, sh_type) : cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2 elif (section == ".symtab" or section == ".dynsym") : - cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \ - " > " + tmpFile1 - cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \ - " > " + tmpFile2 + cmd1 = (elfdump_cmd + " -s -N " + section + " " + f1 + + " > " + tmpFile1) + cmd2 = (elfdump_cmd + " -s -N " + section + " " + f2 + + " > " + tmpFile2) elif (section in text_sections) : # dis sometimes complains when it hits something it doesn't # know how to disassemble. Just ignore it, as the output # being generated here is human readable, and we've already # correctly flagged the difference. - cmd1 = dis_cmd + " -t " + section + " " + f1 + \ - " 2>/dev/null | grep -v disassembly > " + tmpFile1 - cmd2 = dis_cmd + " -t " + section + " " + f2 + \ - " 2>/dev/null | grep -v disassembly > " + tmpFile2 + cmd1 = (dis_cmd + " -t " + section + " " + f1 + + " 2>/dev/null | grep -v disassembly > " + tmpFile1) + cmd2 = (dis_cmd + " -t " + section + " " + f2 + + " 2>/dev/null | grep -v disassembly > " + tmpFile2) else : - cmd1 = elfdump_cmd + " -w " + tmpFile1 + " -N " + \ - section + " " + f1 - cmd2 = elfdump_cmd + " -w " + tmpFile2 + " -N " + \ - section + " " + f2 + cmd1 = (elfdump_cmd + " -w " + tmpFile1 + " -N " + + section + " " + f1) + cmd2 = (elfdump_cmd + " -w " + tmpFile2 + " -N " + + section + " " + f2) os.system(cmd1) os.system(cmd2) @@ -781,12 +785,12 @@ def diff_elf_section(f1, f2, section, sh_type) : # remove temp files as we no longer need them try: os.unlink(tmpFile1) - except OSError, e: - error("diff_elf_section: unlink failed %s" % e) + except OSError as e: + error("diff_elf_section: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("diff_elf_section: unlink failed %s" % e) + except OSError as e: + error("diff_elf_section: unlink failed %s" % e) return (data) @@ -830,7 +834,7 @@ def compareElfs(base, ptch, quiet) : base_header = get_elfheader(base) except: return - sections = base_header.keys() + sections = base_header.keys() try: ptch_header = get_elfheader(ptch) @@ -859,25 +863,25 @@ def compareElfs(base, ptch, quiet) : slist = "" for sect in e1_only_sections : slist = slist + sect + "\t" - data = "ELF sections found in " + \ - base + " but not in " + ptch + \ - "\n\n" + slist + data = ("ELF sections found in " + + base + " but not in " + ptch + + "\n\n" + slist) difference(fileName, "ELF", data) return 1 - + if len(e2_only_sections) > 0 : if quiet : return 1 - + data = "" if logging : slist = "" for sect in e2_only_sections : slist = slist + sect + "\t" - data = "ELF sections found in " + \ - ptch + " but not in " + base + \ - "\n\n" + slist + data = ("ELF sections found in " + + ptch + " but not in " + base + + "\n\n" + slist) difference(fileName, "ELF", data) return 1 @@ -909,7 +913,7 @@ def compareElfs(base, ptch, quiet) : if len(s1) != len (s2) or s1 != s2: if not quiet: sh_type = base_header[sect] - data = diff_elf_section(base, ptch, \ + data = diff_elf_section(base, ptch, sect, sh_type) # If all ELF sections are being reported, then @@ -917,10 +921,10 @@ def compareElfs(base, ptch, quiet) : # stdout only once. Any other section differences # should be logged to the results file directly if not first_section : - log_difference(fileName, \ + log_difference(fileName, "ELF " + sect, data) else : - difference(fileName, "ELF " + sect, \ + difference(fileName, "ELF " + sect, data) if not reportAllSects : @@ -954,7 +958,7 @@ def compareArchives(base, ptch, fileType) : fileName = fnFormat(base) t = threading.currentThread() - ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() + ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName() # @@ -966,42 +970,44 @@ def compareArchives(base, ptch, fileType) : try: os.makedirs(ArchTmpDir1) - except OSError, e: - error("compareArchives: makedir failed %s" % e) + except OSError as e: + error("compareArchives: makedir failed %s" % e) return -1 try: os.makedirs(ArchTmpDir2) - except OSError, e: - error("compareArchives: makedir failed %s" % e) + except OSError as e: + error("compareArchives: makedir failed %s" % e) return -1 # copy over the objects to the temp areas, and # unpack them baseCmd = "cp -fp " + base + " " + ArchTmpDir1 - status, output = commands.getstatusoutput(baseCmd) - if status != 0 : + try: + output = subprocess.check_output(baseCmd) + except CalledProcessError: error(baseCmd + " failed: " + output) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2 - status, output = commands.getstatusoutput(ptchCmd) - if status != 0 : + try: + output = subprocess.check_output(ptchCmd) + except CalledProcessError: error(ptchCmd + " failed: " + output) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 bname = string.split(fileName, '/')[-1] if fileType == "Java Archive" : - baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \ - "; rm -f " + bname + " META-INF/MANIFEST.MF" - ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \ - "; rm -f " + bname + " META-INF/MANIFEST.MF" + baseCmd = ("cd " + ArchTmpDir1 + "; " + "jar xf " + bname + + "; rm -f " + bname + " META-INF/MANIFEST.MF") + ptchCmd = ("cd " + ArchTmpDir2 + "; " + "jar xf " + bname + + "; rm -f " + bname + " META-INF/MANIFEST.MF") elif fileType == "ELF Object Archive" : - baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \ - bname + "; rm -f " + bname - ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \ - bname + "; rm -f " + bname + baseCmd = ("cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + + bname + "; rm -f " + bname) + ptchCmd = ("cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + + bname + "; rm -f " + bname) else : error("unexpected file type: " + fileType) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) @@ -1026,22 +1032,22 @@ def compareArchives(base, ptch, fileType) : for fn in ptchFlist : if not fn in baseFlist : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " added to " + fileName) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 for fn in baseFlist : if not fn in ptchFlist : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " removed from " + fileName) clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 - differs = compareOneFile((ArchTmpDir1 + fn), \ + differs = compareOneFile((ArchTmpDir1 + fn), (ArchTmpDir2 + fn), True) if differs : - difference(fileName, fileType, \ + difference(fileName, fileType, fn + " in " + fileName + " differs") clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 @@ -1091,13 +1097,13 @@ def compareBasic(base, ptch, quiet, fileType) : if needToSnip : toSnipBegin = string.find(baseData, toSnipBeginStr) if toSnipBegin != -1 : - toSnipEnd = string.find(baseData[toSnipBegin:], \ - toSnipEndStr) + \ - len(toSnipEndStr) - baseData = baseData[:toSnipBegin] + \ - baseData[toSnipBegin + toSnipEnd:] - ptchData = ptchData[:toSnipBegin] + \ - ptchData[toSnipBegin + toSnipEnd:] + toSnipEnd = (string.find(baseData[toSnipBegin:], + toSnipEndStr) + + len(toSnipEndStr)) + baseData = (baseData[:toSnipBegin] + + baseData[toSnipBegin + toSnipEnd:]) + ptchData = (ptchData[:toSnipBegin] + + ptchData[toSnipBegin + toSnipEnd:]) if quiet : if baseData != ptchData : @@ -1126,20 +1132,20 @@ def compareByDumping(base, ptch, quiet, fileType) : tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() if fileType == "Lint Library" : - baseCmd = lintdump_cmd + " -ir " + base + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + \ - " | grep -v PASS[1-3]:" + \ - " > " + tmpFile1 - ptchCmd = lintdump_cmd + " -ir " + ptch + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + \ - " | grep -v PASS[1-3]:" + \ - " > " + tmpFile2 + baseCmd = (lintdump_cmd + " -ir " + base + + " | egrep -v '(LINTOBJ|LINTMOD):'" + + " | grep -v PASS[1-3]:" + + " > " + tmpFile1) + ptchCmd = (lintdump_cmd + " -ir " + ptch + + " | egrep -v '(LINTOBJ|LINTMOD):'" + + " | grep -v PASS[1-3]:" + + " > " + tmpFile2) elif fileType == "Sqlite Database" : - baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \ - tmpFile1 - ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \ - tmpFile2 - + baseCmd = ("echo .dump | " + sqlite_cmd + base + " > " + + tmpFile1) + ptchCmd = ("echo .dump | " + sqlite_cmd + ptch + " > " + + tmpFile2) + os.system(baseCmd) os.system(ptchCmd) @@ -1165,24 +1171,24 @@ def compareByDumping(base, ptch, quiet, fileType) : data = diffFileData(tmpFile1, tmpFile2); try: os.unlink(tmpFile1) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) difference(fileName, fileType, data) - return 1 + return 1 # Remove the temporary files now. try: os.unlink(tmpFile1) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) try: os.unlink(tmpFile2) - except OSError, e: - error("compareByDumping: unlink failed %s" % e) + except OSError as e: + error("compareByDumping: unlink failed %s" % e) return 0 @@ -1194,7 +1200,7 @@ def compareByDumping(base, ptch, quiet, fileType) : def discontinue_processing(signl, frme): global keep_processing - print >> sys.stderr, "Caught Ctrl-C, stopping the threads" + print("Caught Ctrl-C, stopping the threads", file=sys.stderr) keep_processing = False return 0 @@ -1204,29 +1210,29 @@ def discontinue_processing(signl, frme): # worker thread for changedFiles processing # class workerThread(threading.Thread) : - def run(self): - global wset_lock - global changedFiles - global baseRoot - global ptchRoot - global keep_processing - - while (keep_processing) : - # grab the lock to changedFiles and remove one member - # and process it - wset_lock.acquire() - try : - fn = changedFiles.pop() - except IndexError : - # there is nothing more to do + def run(self): + global wset_lock + global changedFiles + global baseRoot + global ptchRoot + global keep_processing + + while (keep_processing) : + # grab the lock to changedFiles and remove one member + # and process it + wset_lock.acquire() + try : + fn = changedFiles.pop() + except IndexError : + # there is nothing more to do + wset_lock.release() + return wset_lock.release() - return - wset_lock.release() - base = baseRoot + fn - ptch = ptchRoot + fn + base = baseRoot + fn + ptch = ptchRoot + fn - compareOneFile(base, ptch, False) + compareOneFile(base, ptch, False) ##### @@ -1278,10 +1284,10 @@ def cleanup(ret) : debug("Performing cleanup (" + str(ret) + ")") if os.path.isdir(tmpDir1) > 0 : shutil.rmtree(tmpDir1) - + if os.path.isdir(tmpDir2) > 0 : shutil.rmtree(tmpDir2) - + if logging : log.close() @@ -1352,13 +1358,13 @@ def main() : if sorted : differentFiles = [] - # + # # Build paths to the tools required tools # # Try to look for tools in $SRC/tools if the "-t" option # was specified # - arch = commands.getoutput("uname -p") + arch = subprocess.check_output("uname -p") if localTools : try: src = os.environ['SRC'] @@ -1402,8 +1408,7 @@ def main() : sys.exit(1) if not os.path.exists(ptchRoot) : - error("new proto area: " + ptchRoot + \ - " does not exist") + error("new proto area: " + ptchRoot + " does not exist") sys.exit(1) # @@ -1422,12 +1427,12 @@ def main() : tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/" try: os.makedirs(tmpDir1) - except OSError, e: - error("main: makedir failed %s" % e) + except OSError as e: + error("main: makedir failed %s" % e) try: os.makedirs(tmpDir2) - except OSError, e: - error("main: makedir failed %s" % e) + except OSError as e: + error("main: makedir failed %s" % e) # Derive a catalog of new, deleted, and to-be-compared objects # either from the specified base and patch proto areas, or from @@ -1471,7 +1476,7 @@ def main() : # the appropriate comparison routine, where the compare # will happen, and any differences will be reported / logged - # determine maximum number of worker threads by using + # determine maximum number of worker threads by using # DMAKE_MAX_JOBS environment variable set by nightly(1) # or get number of CPUs in the system try: @@ -1527,4 +1532,3 @@ if __name__ == '__main__' : except KeyboardInterrupt : cleanup(1); - |