From bd47802c98f30f67b323b0796ff5d79a5e308c08 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 20 Apr 2009 19:26:05 +0200 Subject: * utils/migrate-0.8.py: Helper to check Python code for deprecated functions, attributes, etc. Has to be run from the python-apt source tree, but can be used for all Python code using python-apt. The output may not be completely correct, but false positives are better than not checking the code. --- utils/migrate-0.8.py | 194 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100755 utils/migrate-0.8.py (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py new file mode 100755 index 00000000..b3f143b0 --- /dev/null +++ b/utils/migrate-0.8.py @@ -0,0 +1,194 @@ +#!/usr/bin/python2.6 +# +# migrate-0.8.py - Find use of deprecated methods/attributes in the code. +# +# Copyright (C) 2009 Julian Andres Klode +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +# USA +"""migrate-0.8.py - Find all occurences of funcs./attrs. deprecated in 0.8. + +Usage: python2.6 migrate-0.8.py [options] ... + +This reads the list of all functions and attributes available only in +COMPAT_0_7 builds and checks for occurences in the given Python modules. Has +to be run from the python-apt source code directory. + +Requires python2.6 to be installed. + +Parameters: + -h Display this help + -c Colorize the matching parts in the output. +""" +import _ast +import ast +import glob +import linecache +import os +import re +import sys +import types +from collections import defaultdict +from textwrap import fill + +color=False +if sys.argv[1] in ('-c', '--color', '--colour'): + color=True + del sys.argv[1] + +if '-h' in sys.argv or '--help' in sys.argv or not sys.argv[1:]: + print __doc__.strip() + sys.exit(0) + +if os.path.dirname(__file__).endswith('utils'): + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + + +def do_color(string, words): + """Colorize (red) the given words in the given string.""" + if not color: + return string + for word in words: + string = re.sub('([^_]*)(%s)([^_]*)' % word, "\\1\033[31m\033[1m" + + r"\2" + "\033[0m\\3", string) + return string + + +def find_deprecated_cpp(): + """Find all the deprecated functions and attributes.""" + is_open=False + all_old = set() + for fname in glob.glob('python/*.cc'): + lines = list(open(fname, 'r')) + while lines: + line = lines.pop(0) + while lines and not ('static PyMethodDef' in line or + 'static PyGetSetDef' in line): + line = lines.pop(0) + if not lines: + break + + while lines and not ';' in line: + while lines and not 'COMPAT_0_7' in line: + line = lines.pop(0) + if lines: + line = lines.pop(0) + while lines and not '#endif' in line: + all_old.add(line.split(",")[0].strip().strip('{"')) + line = lines.pop(0) + return all_old + + +def find_deprecated_py(): + """Same as find_deprecated_cpp(), but for apt and aptsources. + + We import apt_pkg, set _COMPAT_0_7 to 0, import apt and aptsources and + create a list of all attributes. + + No we remove the imported modules, reimport them (with _COMPAT_0_7=1), + and see which functions have been removed. + """ + + modules = ('apt', 'apt.package', 'apt.cdrom', 'apt.cache', 'apt.debfile', + 'apt.progress', 'aptsources.distinfo', 'aptsources.distro', + 'aptsources.sourceslist') + + import apt_pkg + apt_pkg._COMPAT_0_7 = 0 + + empty = set(sys.modules) + new, deprecated = set(), set() + + for mname in sorted(modules): + module = __import__(mname) + + for clsname in dir(module): + cls = getattr(module, clsname) + if not isinstance(cls, types.TypeType): + new.add(clsname) + continue + new.update(dir(cls)) + + for mname in sys.modules.keys(): + if not mname in empty: + del sys.modules[mname] + + apt_pkg._COMPAT_0_7 = 1 + + for mname in sorted(modules): + module = __import__(mname) + for clsname in dir(module): + cls = getattr(module, clsname) + if not isinstance(cls, types.TypeType): + deprecated.add(clsname) + continue + deprecated.update(dir(cls)) + + + for mname in sys.modules.keys(): + if not mname in empty: + del sys.modules[mname] + + return deprecated.difference(new) + + +def find_occurences(all_old, files): + """Find all ocurrences in the given Python files.""" + for fname in files: + if fname.endswith('setup3.py') or not fname.endswith('.py'): + continue + + words = defaultdict(lambda: set()) + for i in ast.walk(ast.parse(open(fname).read())): + + if isinstance(i, _ast.ImportFrom): + for alias in i.names: + if alias.name in all_old: + words[i.lineno].add(alias.name) + if isinstance(i, _ast.Name) and i.id in all_old: + words[i.lineno].add(i.id) + + if isinstance(i, _ast.Attribute) and i.attr in all_old: + words[i.lineno].add(i.attr) + + for lineno in sorted(words): + line = do_color(linecache.getline(fname, lineno).rstrip('\n'), + words[lineno]) + print '%s:%s:%s' % (fname, lineno, line) + +# Now, let's find them in the code. + +print __doc__.split("\n")[0] +print +if color: + print fill('Information: The color is not always correct, because we ' + 'simply highlight the matched words (like grep).', 79) + print + +all_old = find_deprecated_cpp() | find_deprecated_py() + + +files = set() +for path in sys.argv[1:]: + if not os.path.exists(path): + raise ValueError('Path does not exist: %s' % path) + if os.path.isfile(path): + files.add(path) + else: + for root, dirs, files_ in os.walk(path): + for fname in files_: + files.add(os.path.normpath(os.path.join(root, fname))) + +find_occurences(all_old, sorted(files)) -- cgit v1.2.3 From 3c833358446a42f1b0e2f81bf23489f539c4b70f Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 20 Apr 2009 19:50:53 +0200 Subject: * utils/migrate-0.8.py: Handle attributes specially, reduces false positives. We now prefix attributes with ., so we do not match global variable names when checking. This should reduce the number of false positives in some applications. --- utils/migrate-0.8.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index b3f143b0..32fc58a3 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -61,6 +61,7 @@ def do_color(string, words): if not color: return string for word in words: + word = re.escape(word) string = re.sub('([^_]*)(%s)([^_]*)' % word, "\\1\033[31m\033[1m" + r"\2" + "\033[0m\\3", string) return string @@ -86,7 +87,10 @@ def find_deprecated_cpp(): if lines: line = lines.pop(0) while lines and not '#endif' in line: - all_old.add(line.split(",")[0].strip().strip('{"')) + name = line.split(",")[0].strip().strip('{"') + if 'module' in fname: + name = '.' + name + all_old.add(name) line = lines.pop(0) return all_old @@ -119,7 +123,7 @@ def find_deprecated_py(): if not isinstance(cls, types.TypeType): new.add(clsname) continue - new.update(dir(cls)) + new.update('.' + name for name in dir(cls)) # Attributes/Methods for mname in sys.modules.keys(): if not mname in empty: @@ -134,7 +138,7 @@ def find_deprecated_py(): if not isinstance(cls, types.TypeType): deprecated.add(clsname) continue - deprecated.update(dir(cls)) + deprecated.update('.' + name for name in dir(cls)) # Attributes/Methods for mname in sys.modules.keys(): @@ -160,7 +164,7 @@ def find_occurences(all_old, files): if isinstance(i, _ast.Name) and i.id in all_old: words[i.lineno].add(i.id) - if isinstance(i, _ast.Attribute) and i.attr in all_old: + if isinstance(i, _ast.Attribute) and ('.' + i.attr in all_old): words[i.lineno].add(i.attr) for lineno in sorted(words): @@ -180,6 +184,7 @@ if color: all_old = find_deprecated_cpp() | find_deprecated_py() + files = set() for path in sys.argv[1:]: if not os.path.exists(path): -- cgit v1.2.3 From 0d2eb441d867ab7c6543fdd69f2a632cd354b0a4 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Fri, 24 Apr 2009 17:18:43 +0200 Subject: * utils/migrate-0.8.py: Fix detection of functions, methods and attributes. There was a problem in find_deprecated_cpp() which added '.' to the module-level functions. Caused by a missing 'not'. --- utils/migrate-0.8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index 32fc58a3..be74cc2f 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -88,7 +88,7 @@ def find_deprecated_cpp(): line = lines.pop(0) while lines and not '#endif' in line: name = line.split(",")[0].strip().strip('{"') - if 'module' in fname: + if not 'module' in fname: name = '.' + name all_old.add(name) line = lines.pop(0) -- cgit v1.2.3 From 9f0511895a1a6a1d82e498a609cb4f237fa9b60c Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Fri, 24 Apr 2009 18:06:30 +0200 Subject: * utils/migrate-0.8.py: Correctly import modules, improve attribute detection In order to import modules from a package, 'fromlist' may not be empty. Therefore, we pass ['*'] now. When attributes where checked, we just checked their names and did not check their classes. This meant that in e.g. Compat-API: A.alpha, B.alpha Clean-API: A.alpha The attribute 'alpha' would not be considered deprecated because it is provided by A.alpha. Now we treat an 'alpha' attribute as deprecated, if at least one class loses it. --- utils/migrate-0.8.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index be74cc2f..0986fb21 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -116,14 +116,14 @@ def find_deprecated_py(): new, deprecated = set(), set() for mname in sorted(modules): - module = __import__(mname) + module = __import__(mname, fromlist=['*']) for clsname in dir(module): cls = getattr(module, clsname) if not isinstance(cls, types.TypeType): new.add(clsname) continue - new.update('.' + name for name in dir(cls)) # Attributes/Methods + new.update(clsname + '.' + name for name in dir(cls)) # Attributes/Methods for mname in sys.modules.keys(): if not mname in empty: @@ -132,14 +132,16 @@ def find_deprecated_py(): apt_pkg._COMPAT_0_7 = 1 for mname in sorted(modules): - module = __import__(mname) + module = __import__(mname, fromlist=['*']) for clsname in dir(module): cls = getattr(module, clsname) if not isinstance(cls, types.TypeType): deprecated.add(clsname) continue - deprecated.update('.' + name for name in dir(cls)) # Attributes/Methods - + for name in dir(cls): + if not clsname + '.' + name in new: + # Attributes/Methods, which are deprecated (not in new). + deprecated.add('.' + name) for mname in sys.modules.keys(): if not mname in empty: -- cgit v1.2.3 From 5729c2d5ad92a090cea8648dcddc181a86c09132 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Fri, 24 Apr 2009 18:21:53 +0200 Subject: * utils/migrate-0.8.py: Add a warning that there may be false positives. --- utils/migrate-0.8.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index 0986fb21..99adf652 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -123,7 +123,8 @@ def find_deprecated_py(): if not isinstance(cls, types.TypeType): new.add(clsname) continue - new.update(clsname + '.' + name for name in dir(cls)) # Attributes/Methods + # Attributes/Methods + new.update(clsname + '.' + name for name in dir(cls)) for mname in sys.modules.keys(): if not mname in empty: @@ -158,7 +159,6 @@ def find_occurences(all_old, files): words = defaultdict(lambda: set()) for i in ast.walk(ast.parse(open(fname).read())): - if isinstance(i, _ast.ImportFrom): for alias in i.names: if alias.name in all_old: @@ -178,6 +178,9 @@ def find_occurences(all_old, files): print __doc__.split("\n")[0] print +print fill('Information: Please verify that the results are correct before ' + 'you modify any code, because there may be false positives.', 79) +print if color: print fill('Information: The color is not always correct, because we ' 'simply highlight the matched words (like grep).', 79) @@ -185,8 +188,6 @@ if color: all_old = find_deprecated_cpp() | find_deprecated_py() - - files = set() for path in sys.argv[1:]: if not os.path.exists(path): -- cgit v1.2.3 From d51b51bbc2c7e5d5109f0aa920d730fa6e75b6fb Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 3 Jun 2009 17:45:06 +0200 Subject: utils/migrate-0.8.py: Fix function detection. --- utils/migrate-0.8.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index 99adf652..61d3865a 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -90,6 +90,8 @@ def find_deprecated_cpp(): name = line.split(",")[0].strip().strip('{"') if not 'module' in fname: name = '.' + name + else: + all_old.add('.' + name) all_old.add(name) line = lines.pop(0) return all_old @@ -186,7 +188,12 @@ if color: 'simply highlight the matched words (like grep).', 79) print -all_old = find_deprecated_cpp() | find_deprecated_py() +all_old = find_deprecated_cpp() + +if not '-P' in sys.argv: + all_old |= find_deprecated_py() +else: + sys.argv.remove('-P') files = set() for path in sys.argv[1:]: -- cgit v1.2.3 From 2805097d61d81c92473f3b8e519230b1b23c9fd5 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 4 Jun 2009 17:33:45 +0200 Subject: utils/migrate-0.8.py: Handle constants in the apt_pkg extension. --- utils/migrate-0.8.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'utils') diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index 61d3865a..dc919e1d 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -94,6 +94,20 @@ def find_deprecated_cpp(): all_old.add('.' + name) all_old.add(name) line = lines.pop(0) + # Let's handle constants in the apt_pkg module + lines = list(open('python/apt_pkgmodule.cc')) + while lines: + while lines and not 'COMPAT_0_7' in line: + line = lines.pop(0) + if lines: + lines.pop(0) + while lines and not '#endif' in line: + if 'PyModule_Add' in line: + name = line.split(",")[1].strip().strip('"') + if name != '_COMPAT_0_7': + all_old.add('.' + name) + all_old.add(name) + line = lines.pop(0) return all_old -- cgit v1.2.3 From 93b79ada84fb8f27c2f9c124b2ea9ed87864d967 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Fri, 24 Jul 2009 19:37:48 +0200 Subject: utils/doclint.py: Add a script to check the documentation. --- utils/doclint.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 utils/doclint.py (limited to 'utils') diff --git a/utils/doclint.py b/utils/doclint.py new file mode 100644 index 00000000..a387b05f --- /dev/null +++ b/utils/doclint.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +# Documentation lint. +# Copyright (C) 2009 Julian Andres Klode +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# This comes without any warranty. +"""Read the pickle file created by sphinx and check it.""" + +from __future__ import with_statement +import cPickle +import os +import sys + + +def handle(filename): + with open(filename) as fobj: + index = cPickle.load(fobj) + + objects = index['descrefs'] + modules = index['modules'] + types = index['desctypes'] + + for modname in modules: + module = __import__(modname, fromlist=["*"]) + + for modmember in objects[modname]: + if not modmember in module.__dict__: + print 'W: Unknown', modname + '.' + modmember + elif types[objects[modname][modmember][1]] == u'class': + if modname + '.' + modmember not in objects: + print 'I: No members', modname + '.' + modmember + continue + for member in objects.get(modname + '.' + modmember): + if not member in dir(module.__dict__[modmember]): + print 'W: Unknown', modname + '.' + modmember + '.' + member + + assert(types[objects[modname+"."+modmember][member][1]] in ('method', 'attribute')) + + all = getattr(module, '__all__', []) + for modmember in dir(module): + if getattr(module.__dict__[modmember], "__module__", modname) != modname: + continue + if isinstance(module.__dict__[modmember], type(module)): + continue + if modmember.startswith("_"): + continue + if not modmember in objects[modname] and (not all or modmember in all): + print 'E: Missing', modname + '.' + modmember + elif not modmember in objects[modname]: + print 'W: Missing', modname + '.' + modmember + elif types[objects[modname][modmember][1]] == u'class': + for member in dir(module.__dict__[modmember]): + if member.startswith("_"): + continue + try: + contin = False + for base in module.__dict__[modmember].__bases__: + if member in dir(base): + contin = True + if contin: + continue + except: + pass + if not member in objects.get(modname + '.' + modmember, ""): + print 'E: Missing', modname + '.' + modmember + '.' + member + + +if __name__ == '__main__': + scriptdir = os.path.dirname(__file__) + parentdir = os.path.join(scriptdir, "..") + directory = os.path.join(parentdir, "doc", "build", "pickle") + directory = os.path.normpath(directory) + sys.path.insert(0, os.path.abspath(parentdir)) + handle(os.path.join(directory, "searchindex.pickle")) -- cgit v1.2.3 From 0401747854d7fc26eb097663822e26a988cf4aa4 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Sun, 7 Feb 2010 20:26:46 +0100 Subject: * utils/migrate-0.8.py: - Improve C++ parsing and add apt.progress.old to the modules, reduces false positives. --- debian/changelog | 3 +++ utils/migrate-0.8.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'utils') diff --git a/debian/changelog b/debian/changelog index d981c8b5..b09fb684 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ python-apt (0.7.93.2) UNRELEASED; urgency=low - apt/utils.py: Completely ported, previous one was old-API from Ubuntu. - apt/cache.py: Use the new progress classes instead of the old ones. - apt/package.py: Various smaller issues fixed, probably caused by merge. + * utils/migrate-0.8.py: + - Improve C++ parsing and add apt.progress.old to the modules, reduces + false positives. -- Julian Andres Klode Sun, 07 Feb 2010 19:58:40 +0100 diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py index dc919e1d..61059b2a 100755 --- a/utils/migrate-0.8.py +++ b/utils/migrate-0.8.py @@ -82,8 +82,10 @@ def find_deprecated_cpp(): break while lines and not ';' in line: - while lines and not 'COMPAT_0_7' in line: + while lines and not 'COMPAT_0_7' in line and not ';' in line: line = lines.pop(0) + if ';' in line: + continue if lines: line = lines.pop(0) while lines and not '#endif' in line: @@ -122,8 +124,8 @@ def find_deprecated_py(): """ modules = ('apt', 'apt.package', 'apt.cdrom', 'apt.cache', 'apt.debfile', - 'apt.progress', 'aptsources.distinfo', 'aptsources.distro', - 'aptsources.sourceslist') + 'apt.progress', 'apt.progress.old', 'aptsources.distinfo', + 'aptsources.distro', 'aptsources.sourceslist') import apt_pkg apt_pkg._COMPAT_0_7 = 0 -- cgit v1.2.3