4 files changed, 320 insertions, 0 deletions
diff --git a/utils/get_debian_mirrors.py b/utils/get_debian_mirrors.py
new file mode 100755
index 00000000..ccddf8ed
--- /dev/null
+++ b/utils/get_debian_mirrors.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+#
+#  get_debian_mirrors.py
+#
+#  Download the latest list with available mirrors from the Debian 
+#  website and extract the hosts from the raw page
+#
+#  Copyright (c) 2006 Free Software Foundation Europe
+#
+#  Author: Sebastian Heinlein <glatzor@ubuntu.com>
+# 
+#  This program is free software; you can redistribute it and/or 
+#  modify it under the terms of the GNU General Public License as 
+#  published by the Free Software Foundation; either version 2 of the
+#  License, or (at your option) any later version.
+# 
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+# 
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+
+import urllib2
+import re
+import os
+import commands
+import sys
+
+# the list of official Ubuntu servers
+mirrors = []
+# path to the local mirror list
+list_path = "../data/templates/Debian.mirrors"
+
+req = urllib2.Request("http://www.debian.org/mirror/mirrors_full")
+match = re.compile("^.*>([A-Za-z0-9-.\/_]+)<\/a>.*\n$")
+match_location = re.compile('^<strong><a name="([A-Z]+)">.*')
+
+def add_sites(line, proto, sites, mirror_type):
+    path = match.sub(r"\1", line)
+    for site in sites:
+        mirror_type.append("%s://%s%s\n" % (proto, site.lstrip(), path))
+
+try:
+    print "Downloading mirrors list from the Debian website..."
+    uri=urllib2.urlopen(req)
+    for line in uri.readlines():
+        if line.startswith('<strong><a name="'):
+            location = match_location.sub(r"\1", line)
+            if location:
+                mirrors.append("#LOC:%s" % location)
+        if line.startswith("Site:"):
+            sites = line[6:-1].split(",")
+        elif line.startswith('Packages over HTTP'):
+            add_sites(line, "http", sites, mirrors)
+        elif line.startswith('Packages over FTP'):
+            add_sites(line, "ftp", sites, mirrors)
+    uri.close()
+except:
+    print "Failed to download or to extract the mirrors list!"
+    sys.exit(1)
+
+print "Writing local mirrors list: %s" % list_path
+list = open(list_path, "w")
+for mirror in mirrors:
+    list.write("%s" % mirror)
+list.close()
+print "Done."
diff --git a/utils/get_ubuntu_mirrors.py b/utils/get_ubuntu_mirrors.py
new file mode 100755
index 00000000..62b18ba6
--- /dev/null
+++ b/utils/get_ubuntu_mirrors.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+#
+#  get_ubuntu_mirrors.py
+#
+#  Download the latest list with available mirrors from the Ubuntu
+#  wiki and extract the hosts from the raw page
+#
+#  Copyright (c) 2006 Free Software Foundation Europe
+#
+#  Author: Sebastian Heinlein <glatzor@ubuntu.com>
+# 
+#  This program is free software; you can redistribute it and/or 
+#  modify it under the terms of the GNU General Public License as 
+#  published by the Free Software Foundation; either version 2 of the
+#  License, or (at your option) any later version.
+# 
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+# 
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+
+import urllib2
+import re
+import os
+import commands
+import sys
+
+# the list of official Ubuntu servers
+mirrors = []
+# path to the local mirror list
+list_path = "../data/templates/Ubuntu.mirrors"
+
+req = urllib2.Request("https://wiki.ubuntu.com/Archive?action=raw")
+
+try:
+    print "Downloading mirrors list from the Ubuntu wiki..."
+    uri=urllib2.urlopen(req)
+    p = re.compile('^.*((http|ftp):\/\/[A-Za-z0-9-.:\/_]+).*\n*$')
+    for line in uri.readlines():
+         if r"[[Anchor(dvd-images)]]" in line:
+             break
+         if "http://" in line or "ftp://" in line:
+             mirrors.append(p.sub(r"\1", line))
+    uri.close()
+except:
+    print "Failed to download or extract the mirrors list!"
+    sys.exit(1)
+
+print "Writing local mirrors list: %s" % list_path
+list = open(list_path, "w")
+for mirror in mirrors:
+    list.write("%s\n" % mirror)
+list.close()
+print "Done."
+
+
diff --git a/utils/get_ubuntu_mirrors_from_lp.py b/utils/get_ubuntu_mirrors_from_lp.py
new file mode 100755
index 00000000..7d9116f2
--- /dev/null
+++ b/utils/get_ubuntu_mirrors_from_lp.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+#
+#  get_ubuntu_lp_mirrors.py
+#
+#  Download the latest list with available Ubuntu mirrors from Launchpad.net
+#  and extract the hosts from the raw page
+#
+#  Copyright (c) 2006 Free Software Foundation Europe
+#
+#  Author: Sebastian Heinlein <glatzor@ubuntu.com>
+# 
+#  This program is free software; you can redistribute it and/or 
+#  modify it under the terms of the GNU General Public License as 
+#  published by the Free Software Foundation; either version 2 of the
+#  License, or (at your option) any later version.
+# 
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+# 
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+#  USA
+
+import urllib2
+import re
+import sys
+
+# the list of official Ubuntu servers
+mirrors = []
+# path to the local mirror list
+list_path = "../data/templates/Ubuntu.mirrors"
+
+
+try:
+    f = open("/usr/share/iso-codes/iso_3166.tab", "r")
+    lines = f.readlines()
+    f.close()
+except:
+    print "Could not read country information"
+    sys.exit(1)
+
+countries = {}
+for line in lines:
+    parts = line.split("\t")
+    countries[parts[1].strip()] = parts[0].lower()
+
+req = urllib2.Request("https://launchpad.net/ubuntu/+archivemirrors")
+print "Downloading mirrors list from Launchpad..."
+try:
+    uri=urllib2.urlopen(req)
+    content = uri.read()
+    uri.close()
+except:
+    print "Failed to download or extract the mirrors list!"
+    sys.exit(1)
+
+content = content.replace("\n", "")
+
+content_splits = re.split(r'<tr class="highlighted"',
+                          re.findall(r'<table class="listing" '
+                                      'id="mirrors_list">.+?</table>',
+                                     content)[0])
+lines=[]
+def find(split):
+    country = re.search(r"<strong>(.+?)</strong>", split)
+    if not country:
+        return
+    if countries.has_key(country.group(1)):
+        lines.append("#LOC:%s" % countries[country.group(1)].upper())
+    else:
+        lines.append("#LOC:%s" % country.group(1))
+    # FIXME: currently the protocols are hardcoded: ftp http
+    urls = re.findall(r'<a href="(?![a-zA-Z:/_\-]+launchpad.+?">)'
+                       '(((http)|(ftp)).+?)">',
+                      split)
+    map(lambda u: lines.append(u[0]), urls)
+
+map(find, content_splits)
+
+print "Writing local mirrors list: %s" % list_path
+list = open(list_path, "w")
+for line in lines:
+    list.write("%s\n" % line)
+list.close()
+print "Done."
diff --git a/utils/mirrortest b/utils/mirrortest
new file mode 100755
index 00000000..75ef917b
--- /dev/null
+++ b/utils/mirrortest
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+import threading, Queue, time, re, os, tempfile
+import aptsources.sourceslist
+import aptsources.distro
+from timeit import Timer
+import urllib
+import socket
+import random
+socket.setdefaulttimeout(2)
+
+class MirrorTest:
+    class PingWorker(threading.Thread):
+        def __init__(self, jobs, results, id):
+            self.id = id
+            self.jobs = jobs
+            self.results = results
+            self.match_result = re.compile(r"^rtt .* = [\.\d]+/([\.\d]+)/.*")
+            threading.Thread.__init__(self)
+        def run(self):
+            result = None
+            while MirrorTest.completed_pings < MirrorTest.todo:
+                try:
+                    mirror = self.jobs.get(True, 1)
+                    host = mirror.hostname
+                except:
+                    continue
+                print "Pinging (Worker %s) %s (%s) ..." % (self.id, 
+                                                           host, 
+                                                           MirrorTest.completed_pings)
+                commando = os.popen("ping -q -c 4 -W 2 -i 0.3 %s" % host,
+                                    "r")
+                while True:
+                    line = commando.readline()
+                    if not line:
+                        break
+                    result = re.findall(self.match_result, line)
+                MirrorTest.completed_pings_lock.acquire()
+                MirrorTest.completed_pings += 1
+                if result:
+                    self.results.append([float(result[0]), host, mirror])
+                MirrorTest.completed_pings_lock.release()
+
+    def speed_test(self, mirror):
+        url = "%s/%s" % (mirror.get_repo_urls()[0],
+                         self.test_file)
+        print "Downloading %s ..." % url
+        start = time.time()
+        try:
+            data = urllib.urlopen(url).read(51200)
+            return 50 / (time.time() - start)
+        except:
+            return 0
+
+    def __init__(self, hosts, test_file):
+        self.test_file = test_file
+        jobs = Queue.Queue()
+        results = []
+        for h in hosts:
+            jobs.put(h)
+        self.threads = []
+        MirrorTest.completed_pings = 0
+        MirrorTest.completed_pings_lock = threading.Lock()
+        MirrorTest.todo = len(hosts)
+
+        for i in range(10):
+            t = MirrorTest.PingWorker(jobs, results, i)
+            self.threads.append(t)
+            t.start()
+
+        for t in self.threads:
+            t.join()
+
+        results.sort()
+        print "\n\nTop ten RTTs:"
+        for r in results[0:10]:
+            print "%s: %s" % (r[1], r[0])
+        print "\n\n"
+
+        results.insert(0, [0, "rand", hosts[random.randint(1, len(hosts))]])
+        results.insert(0, [0, "rand", hosts[random.randint(1, len(hosts))]])
+
+        final = map(lambda r: (self.speed_test(r[2]), r[2]),
+                    results[0:12])
+        final.sort()
+        final.reverse()
+        print "\n\nBest mirrors:"
+        for f in final:
+            print "%s: %s KByte/s" % (f[1].hostname, int(f[0]))
+
+if __name__ == "__main__":
+    distro = aptsources.distro.get_distro()
+    distro.get_sources(aptsources.sourceslist.SourcesList())
+    pipe = os.popen("dpkg --print-architecture")
+    arch = pipe.read().strip()
+    test_file = "dists/%s/%s/binary-%s/Packages.gz" % \
+                (distro.source_template.name,
+                 distro.source_template.components[0].name,
+                 arch)
+    app = MirrorTest(distro.source_template.mirror_set.values(),
+                     test_file)