merged from lp:~mvo/python-apt/mvo

4 files changed, 345 insertions, 58 deletions

diff --git a/utils/doclint.py b/utils/doclint.py
new file mode 100644
index 00000000..a387b05f
--- /dev/null
+++ b/utils/doclint.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# Documentation lint.
+# Copyright (C) 2009 Julian Andres Klode <jak@debian.org>
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.
+#
+# This comes without any warranty.
+"""Read the pickle file created by sphinx and check it."""
+
+from __future__ import with_statement
+import cPickle
+import os
+import sys
+
+
+def handle(filename):
+    with open(filename) as fobj:
+        index = cPickle.load(fobj)
+
+        objects = index['descrefs']
+        modules = index['modules']
+        types = index['desctypes']
+
+    for modname in modules:
+        module = __import__(modname, fromlist=["*"])
+
+        for modmember in objects[modname]:
+            if not modmember in module.__dict__:
+                print 'W: Unknown', modname + '.' + modmember
+            elif types[objects[modname][modmember][1]] == u'class':
+                if modname + '.' + modmember not in objects:
+                    print 'I: No members', modname + '.' + modmember
+                    continue
+                for member in objects.get(modname + '.' + modmember):
+                    if not member in dir(module.__dict__[modmember]):
+                        print 'W: Unknown', modname + '.' + modmember + '.' + member
+
+                    assert(types[objects[modname+"."+modmember][member][1]] in ('method', 'attribute'))
+
+        all = getattr(module, '__all__', [])
+        for modmember in dir(module):
+            if getattr(module.__dict__[modmember], "__module__", modname) != modname:
+                continue
+            if isinstance(module.__dict__[modmember], type(module)):
+                continue
+            if modmember.startswith("_"):
+                continue
+            if not modmember in objects[modname] and (not all or modmember in all):
+                print 'E: Missing', modname + '.' + modmember
+            elif not modmember in objects[modname]:
+                print 'W: Missing', modname + '.' + modmember
+            elif types[objects[modname][modmember][1]] == u'class':
+                for member in dir(module.__dict__[modmember]):
+                    if member.startswith("_"):
+                        continue
+                    try:
+                        contin = False
+                        for base in module.__dict__[modmember].__bases__:
+                            if member in dir(base):
+                                contin = True
+                        if contin:
+                            continue
+                    except:
+                        pass
+                    if not member in objects.get(modname + '.' + modmember, ""):
+                        print 'E: Missing', modname + '.' + modmember + '.' + member
+
+
+if __name__ == '__main__':
+    scriptdir = os.path.dirname(__file__)
+    parentdir = os.path.join(scriptdir, "..")
+    directory = os.path.join(parentdir,  "doc", "build", "pickle")
+    directory = os.path.normpath(directory)
+    sys.path.insert(0, os.path.abspath(parentdir))
+    handle(os.path.join(directory, "searchindex.pickle"))
diff --git a/utils/get_debian_mirrors.py b/utils/get_debian_mirrors.py
index 9fb783bd..395c21da 100755
--- a/utils/get_debian_mirrors.py
+++ b/utils/get_debian_mirrors.py
@@ -1,13 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/python
+#  get_debian_mirrors.py - Parse Mirrors.masterlist and create a mirror list.
 #
-#  get_debian_mirrors.py
-#
-#  Download the latest list with available mirrors from the Debian
-#  website and extract the hosts from the raw page
-#
-#  Copyright (c) 2006, 2009 Free Software Foundation Europe
-#
-#  Author: Sebastian Heinlein <glatzor@ubuntu.com>
+#  Copyright (c) 2010 Julian Andres Klode <jak@debian.org>
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU General Public License as
@@ -23,52 +17,22 @@
 #  along with this program; if not, write to the Free Software
 #  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 #  USA
-
+import collections
 import urllib2
-import re
-import os
-import commands
-import sys
-
-# the list of official Ubuntu servers
-mirrors = []
-# path to the local mirror list
-list_path = "../data/templates/Debian.mirrors"
-
-req = urllib2.Request("http://www.debian.org/mirror/mirrors_full")
-match = re.compile("^.*>([A-Za-z0-9-.\/_]+)<\/a>.*\n$")
-match_location = re.compile('^<h3><a name="([A-Z]+)">.*')
-match_sites = re.compile('^Site: <tt>([A-Za-z0-9-.\ \/_,]+)<\/tt>.*\n$')
-
-
-def add_sites(line, proto, sites, mirror_type):
-    path = match.sub(r"\1", line)
-    for site in sites:
-        mirror_type.append("%s://%s%s\n" % (proto, site.lstrip(), path))
-
-
-try:
-    print "Downloading mirrors list from the Debian website..."
-    uri=urllib2.urlopen(req)
-    for line in uri.readlines():
-        if line.startswith('<h3><a name="'):
-            location = match_location.sub(r"\1", line)
-            if location:
-                mirrors.append("#LOC:%s" % location)
-        if line.startswith("Site:"):
-            sites = match_sites.sub(r"\1", line).split(",")
-        elif line.startswith('<br>Packages over HTTP'):
-            add_sites(line, "http", sites, mirrors)
-        elif line.startswith('<br>Packages over FTP'):
-            add_sites(line, "ftp", sites, mirrors)
-    uri.close()
-except:
-    print "Failed to download or to extract the mirrors list!"
-    sys.exit(1)
-
-print "Writing local mirrors list: %s" % list_path
-list = open(list_path, "w")
-for mirror in mirrors:
-    list.write("%s" % mirror)
-list.close()
-print "Done."
+from debian_bundle import deb822
+
+mirrors = collections.defaultdict(set)
+masterlist = urllib2.urlopen("http://cvs.debian.org/webwml/webwml/english/"
+                             "mirror/Mirrors.masterlist?revision=HEAD")
+
+for mirror in deb822.Deb822.iter_paragraphs(masterlist):
+    country = mirror["Country"].split(None, 1)[0]
+    site = mirror["Site"]
+    for proto in 'http', 'ftp':
+        if "Archive-%s" % proto in mirror:
+            mirrors[country].add("%s://%s%s" % (proto, site,
+                                                mirror["Archive-%s" % proto]))
+
+for country in sorted(mirrors):
+    print "#LOC:%s" % country
+    print "\n".join(sorted(mirrors[country]))
diff --git a/utils/get_ubuntu_mirrors_from_lp.py b/utils/get_ubuntu_mirrors_from_lp.py
index 3c183b6d..341dba8a 100755
--- a/utils/get_ubuntu_mirrors_from_lp.py
+++ b/utils/get_ubuntu_mirrors_from_lp.py
@@ -44,4 +44,4 @@ keys = countries.keys()
 keys.sort()
 for country in keys:
     print "#LOC:%s" % country
-    print "\n".join(countries[country])
+    print "\n".join(sorted(countries[country]))
diff --git a/utils/migrate-0.8.py b/utils/migrate-0.8.py
new file mode 100755
index 00000000..d0d8e9a1
--- /dev/null
+++ b/utils/migrate-0.8.py
@@ -0,0 +1,246 @@
+#!/usr/bin/python2.6
+#
+# migrate-0.8.py - Find use of deprecated methods/attributes in the code.
+#
+# Copyright (C) 2009 Julian Andres Klode <jak@debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+"""migrate-0.8.py - Find all occurences of funcs./attrs. deprecated in 0.8.
+
+Usage: python2.6 migrate-0.8.py [options] <file/directory>...
+
+This reads the list of all functions and attributes available only in
+COMPAT_0_7 builds and checks for occurences in the given Python modules. Has
+to be run from the python-apt source code directory.
+
+Requires python2.6 to be installed.
+
+Parameters:
+    -h  Display this help
+    -c  Colorize the matching parts in the output.
+"""
+import _ast
+import ast
+import glob
+import linecache
+import os
+import re
+import sys
+import types
+from collections import defaultdict
+from textwrap import fill
+
+color=False
+if len(sys.argv) > 1 and sys.argv[1] in ('-c', '--color', '--colour'):
+    color=True
+    del sys.argv[1]
+
+if '-h' in sys.argv or '--help' in sys.argv or not sys.argv[1:]:
+    print __doc__.strip()
+    sys.exit(0)
+
+if os.path.dirname(__file__).endswith('utils'):
+    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+
+deprecated_cpp_stuff = set([
+    '.Add', '.AllTargets', '.Arch', '.Architecture', '.Archive', 
+    '.ArchiveURI', '.Auto', '.Base64Encode', '.Binaries', '.BrokenCount', 
+    '.BuildDepends', '.Bytes', '.CheckDep', '.CheckDomainList', '.Clear', 
+    '.Close', '.Commit', '.CompType', '.Complete', '.Component', '.Config', 
+    '.CurStateConfigFiles', '.CurStateHalfConfigured', 
+    '.CurStateHalfInstalled', '.CurStateInstalled', '.CurStateNotInstalled', 
+    '.CurStateUnPacked', '.CurrentState', '.CurrentVer', '.Date', 
+    '.DeQuoteString', '.DebSize', '.DelCount', '.DepType', '.DependsCount', 
+    '.DependsList', '.DependsListStr', '.DescURI', '.Describe', '.DestFile', 
+    '.Dist', '.DoInstall', '.Downloadable', '.ErrorText', '.Essential', 
+    '.Exists', '.FetchNeeded', '.FileList', '.FileName', '.FileSize', 
+    '.Files', '.Find', '.FindB', '.FindDir', '.FindFile', '.FindFlag', 
+    '.FindI', '.FindIndex', '.FindRaw', '.FixBroken', '.FixMissing', 
+    '.GetAcquire', '.GetArchives', '.GetCache', '.GetCandidateVer', 
+    '.GetCdrom', '.GetDepCache', '.GetIndexes', '.GetLock', 
+    '.GetPackageManager', '.GetPkgAcqFile', '.GetPkgActionGroup', 
+    '.GetPkgProblemResolver', '.GetPkgRecords', '.GetPkgSourceList', 
+    '.GetPkgSrcRecords', '.HasPackages', '.Hash', '.Homepage', '.ID', 
+    '.Ident', '.Important', '.Index', '.IndexFiles', '.IndexType', '.Init', 
+    '.InitConfig', '.InitSystem', '.InstCount', '.InstState', 
+    '.InstStateHold', '.InstStateHoldReInstReq', '.InstStateOk', 
+    '.InstStateReInstReq', '.InstallProtect', '.InstalledSize', 
+    '.IsAutoInstalled', '.IsGarbage', '.IsInstBroken', '.IsNowBroken', 
+    '.IsTrusted', '.IsUpgradable', '.Items', '.Jump', '.KeepCount', 
+    '.Label', '.LanguageCode', '.LibVersion', '.List', '.Local', 
+    '.LongDesc', '.Lookup', '.MD5Hash', '.Maintainer', '.MarkDelete', 
+    '.MarkInstall', '.MarkKeep', '.MarkedDelete', '.MarkedDowngrade', 
+    '.MarkedInstall', '.MarkedKeep', '.MarkedReinstall', '.MarkedUpgrade', 
+    '.MinimizeUpgrade', '.MyTag', '.Name', '.NotAutomatic', '.NotSource', 
+    '.Offset', '.Open', '.Origin', '.Package', '.PackageCount', 
+    '.PackageFileCount', '.Packages', '.ParentPkg', '.ParentVer', 
+    '.ParseCommandLine', '.ParseDepends', '.ParseSection', 
+    '.ParseSrcDepends', '.ParseTagFile', '.PartialPresent', 
+    '.PkgSystemLock', '.PkgSystemUnLock', '.PriExtra', '.PriImportant', 
+    '.PriOptional', '.PriRequired', '.PriStandard', '.Priority', 
+    '.PriorityStr', '.Protect', '.ProvidesCount', '.ProvidesList', 
+    '.QuoteString', '.ReadConfigDir', '.ReadConfigFile', 
+    '.ReadConfigFileISC', '.ReadMainList', '.ReadPinFile', '.Record', 
+    '.Remove', '.Resolve', '.ResolveByKeep', '.Restart', '.RevDependsList', 
+    '.RewriteSection', '.RewriteSourceOrder', '.Run', '.SHA1Hash', 
+    '.SHA256Hash', '.Section', '.SelStateDeInstall', '.SelStateHold', 
+    '.SelStateInstall', '.SelStatePurge', '.SelStateUnknown', 
+    '.SelectedState', '.Set', '.SetCandidateVer', '.SetReInstall', 
+    '.ShortDesc', '.Shutdown', '.Site', '.Size', '.SizeToStr', 
+    '.SmartTargetPkg', '.SourcePkg', '.SourceVer', '.Status', '.Step', 
+    '.StrToTime', '.StringToBool', '.SubTree', '.TargetPkg', '.TargetVer', 
+    '.Time', '.TimeRFC1123', '.TimeToStr', '.TotalNeeded', 
+    '.TranslationDescription', '.URI', '.URItoFileName', '.Update', 
+    '.Upgrade', '.UpstreamVersion', '.UsrSize', '.ValueList', 
+    '.VerFileCount', '.VerStr', '.Version', '.VersionCompare', 
+    '.VersionCount', '.VersionList', '.newConfiguration', 'Base64Encode', 
+    'CheckDep', 'CheckDomainList', 'Config', 'CurStateConfigFiles', 
+    'CurStateHalfConfigured', 'CurStateHalfInstalled', 'CurStateInstalled', 
+    'CurStateNotInstalled', 'CurStateUnPacked', 'Date', 'DeQuoteString', 
+    'GetAcquire', 'GetCache', 'GetCdrom', 'GetDepCache', 'GetLock', 
+    'GetPackageManager', 'GetPkgAcqFile', 'GetPkgActionGroup', 
+    'GetPkgProblemResolver', 'GetPkgRecords', 'GetPkgSourceList', 
+    'GetPkgSrcRecords', 'InitConfig', 'InitSystem', 'InstStateHold', 
+    'InstStateHoldReInstReq', 'InstStateOk', 'InstStateReInstReq', 
+    'LibVersion', 'ParseCommandLine', 'ParseDepends', 'ParseSection', 
+    'ParseSrcDepends', 'ParseTagFile', 'PkgSystemLock', 'PkgSystemUnLock', 
+    'PriExtra', 'PriImportant', 'PriOptional', 'PriRequired', 'PriStandard', 
+    'QuoteString', 'ReadConfigDir', 'ReadConfigFile', 'ReadConfigFileISC', 
+    'RewriteSection', 'RewriteSourceOrder', 'SelStateDeInstall', 
+    'SelStateHold', 'SelStateInstall', 'SelStatePurge', 'SelStateUnknown', 
+    'SizeToStr', 'StrToTime', 'StringToBool', 'Time', 'TimeRFC1123', 
+    'TimeToStr', 'URItoFileName', 'UpstreamVersion', 'VersionCompare', 
+    'newConfiguration'])
+
+def do_color(string, words):
+    """Colorize (red) the given words in the given string."""
+    if not color:
+        return string
+    for word in words:
+        word = re.escape(word)
+        string = re.sub('([^_]*)(%s)([^_]*)' % word, "\\1\033[31m\033[1m" +
+                        r"\2" + "\033[0m\\3", string)
+    return string
+
+def find_deprecated_py():
+    """Find all the deprecated functions and attributes.
+
+    Import apt_pkg, set _COMPAT_0_7 to 0, import apt and aptsources and
+    create a list of all attributes. Then remove the imported modules,
+    reimport them (with _COMPAT_0_7=1), and see which functions do not
+    exist anymore.
+    """
+
+    modules = ('apt', 'apt.package', 'apt.cdrom', 'apt.cache', 'apt.debfile',
+               'apt.progress', 'apt.progress.old', 'aptsources.distinfo',
+               'aptsources.distro', 'aptsources.sourceslist')
+
+    import apt_pkg
+    apt_pkg._COMPAT_0_7 = 0
+
+    empty = set(sys.modules)
+    new, deprecated = set(), set()
+
+    for mname in sorted(modules):
+        module = __import__(mname, fromlist=['*'])
+
+        for clsname in dir(module):
+            cls = getattr(module, clsname)
+            if not isinstance(cls, types.TypeType):
+                new.add(clsname)
+                continue
+            # Attributes/Methods
+            new.update(clsname + '.' + name for name in dir(cls))
+
+    for mname in sys.modules.keys():
+        if not mname in empty:
+            del sys.modules[mname]
+
+    apt_pkg._COMPAT_0_7 = 1
+
+    for mname in sorted(modules):
+        module = __import__(mname, fromlist=['*'])
+        for clsname in dir(module):
+            cls = getattr(module, clsname)
+            if not isinstance(cls, types.TypeType):
+                deprecated.add(clsname)
+                continue
+            for name in dir(cls):
+                if not clsname + '.' + name in new:
+                    # Attributes/Methods, which are deprecated (not in new).
+                    deprecated.add('.' + name)
+
+    for mname in sys.modules.keys():
+        if not mname in empty:
+            del sys.modules[mname]
+
+    return deprecated.difference(new)
+
+
+def find_occurences(all_old, files):
+    """Find all ocurrences in the given Python files."""
+    for fname in files:
+        if fname.endswith('setup3.py') or not fname.endswith('.py'):
+            continue
+
+        words = defaultdict(lambda: set())
+        for i in ast.walk(ast.parse(open(fname).read())):
+            if isinstance(i, _ast.ImportFrom):
+                for alias in i.names:
+                    if alias.name in all_old:
+                        words[i.lineno].add(alias.name)
+            if isinstance(i, _ast.Name) and i.id in all_old:
+                words[i.lineno].add(i.id)
+
+            if isinstance(i, _ast.Attribute) and ('.' + i.attr in all_old):
+                words[i.lineno].add(i.attr)
+
+        for lineno in sorted(words):
+            line = do_color(linecache.getline(fname, lineno).rstrip('\n'),
+                            words[lineno])
+            print '%s:%s:%s' % (fname, lineno, line)
+
+# Now, let's find them in the code.
+
+print __doc__.split("\n")[0]
+print
+print fill('Information: Please verify that the results are correct before '
+           'you modify any code, because there may be false positives.', 79)
+print
+if color:
+    print fill('Information: The color is not always correct, because we '
+               'simply highlight the matched words (like grep).', 79)
+    print
+
+all_old = deprecated_cpp_stuff
+
+if not '-P' in sys.argv:
+    all_old |= find_deprecated_py()
+else:
+    sys.argv.remove('-P')
+
+files = set()
+for path in sys.argv[1:]:
+    if not os.path.exists(path):
+        raise ValueError('Path does not exist: %s' % path)
+    if os.path.isfile(path):
+        files.add(path)
+    else:
+        for root, dirs, files_ in os.walk(path):
+            for fname in files_:
+                files.add(os.path.normpath(os.path.join(root, fname)))
+
+find_occurences(all_old, sorted(files))