* apt/progress/__init__.py:

  - add update_status_full() that takes file_size/partial_size as 
    additional callback arguments
  - add pulse_items() that takes a addtional "items" tuple that 
    gives the user full access to the individual items that are
    fetched
* python/progress.cc:
  - low level code for update_status_full and pulse_items()
  - better threading support

* aptsources/distro.py:
  - fix indent error that causes incorrect sources.list additons
    (LP: #372224)
* python/progress.cc:
  - fix crash in RunSimpleCallback()
* apt/cache.py:
  - when the cache is run with a alternative rootdir, create
    required dirs/files automatically

9 files changed, 266 insertions, 21 deletions

diff --git a/apt/cache.py b/apt/cache.py
index cc425ccb..94a77fd8 100644
--- a/apt/cache.py
+++ b/apt/cache.py
@@ -60,8 +60,33 @@ class Cache(object):
             apt_pkg.Config.Set("Dir", rootdir)
             apt_pkg.Config.Set("Dir::State::status",
                                rootdir + "/var/lib/dpkg/status")
+            # create required dirs/files when run with special rootdir
+            # automatically
+            self._check_and_create_required_dirs(rootdir)
         self.open(progress)
 
+    def _check_and_create_required_dirs(self, rootdir):
+        """
+        check if the required apt directories/files are there and if
+        not create them
+        """
+        files = ["/var/lib/dpkg/status",
+                 "/etc/apt/sources.list",
+                ]
+        dirs = ["/var/lib/dpkg",
+                "/etc/apt/",
+                "/var/cache/apt/archives/partial",
+                "/var/lib/apt/lists/partial",
+               ]
+        for d in dirs:
+            if not os.path.exists(rootdir+d):
+                print "creating: ",rootdir+d
+                os.makedirs(rootdir+d)
+        for f in files:
+            if not os.path.exists(rootdir+f):
+                open(rootdir+f,"w")
+       
+
     def _runCallbacks(self, name):
         """ internal helper to run a callback """
         if name in self._callbacks:
diff --git a/apt/progress/__init__.py b/apt/progress/__init__.py
index 769942ce..3be197f7 100644
--- a/apt/progress/__init__.py
+++ b/apt/progress/__init__.py
@@ -112,6 +112,13 @@ class FetchProgress(object):
 
         This happens eg. when the downloads fails or is completed.
         """
+    def update_status_full(self, uri, descr, short_descr, status, file_size, 
+                           partial_size):
+        """Called when the status of an item changes.
+
+        This happens eg. when the downloads fails or is completed. This
+        version include information on current filesize and partial size
+        """
 
     def pulse(self):
         """Called periodically to update the user interface.
@@ -125,6 +132,19 @@ class FetchProgress(object):
                         float(self.currentCPS))
         return True
 
+    def pulse_items(self, items):
+        """Called periodically to update the user interface.
+        This function includes details about the items being fetched
+        Return True to continue or False to cancel.
+
+        """
+        self.percent = (((self.currentBytes + self.currentItems) * 100.0) /
+                        float(self.totalBytes + self.totalItems))
+        if self.currentCPS > 0:
+            self.eta = ((self.totalBytes - self.currentBytes) /
+                        float(self.currentCPS))
+        return True
+
     def mediaChange(self, medium, drive):
         """react to media change events."""
 
diff --git a/aptsources/distro.py b/aptsources/distro.py
index bbb8ba50..5398d4a3 100644
--- a/aptsources/distro.py
+++ b/aptsources/distro.py
@@ -319,12 +319,12 @@ class Distribution:
             if s.type == self.binary_type:
                 if s.dist not in comps_per_dist:
                     comps_per_dist[s.dist] = set()
-                    map(comps_per_dist[s.dist].add, s.comps)
+                map(comps_per_dist[s.dist].add, s.comps)
         for s in self.source_code_sources:
             if s.type == self.source_type:
                 if s.dist not in comps_per_sdist:
                     comps_per_sdist[s.dist] = set()
-                    map(comps_per_sdist[s.dist].add, s.comps)
+                map(comps_per_sdist[s.dist].add, s.comps)
 
         # check if there is a main source at all
         if len(self.main_sources) < 1:
diff --git a/debian/changelog b/debian/changelog
index b3b811c2..0e4731f6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,9 +1,31 @@
-python-apt (0.7.10.5) UNRELEASED; urgency=low
+python-apt (0.7.11.0) UNRELEASED; urgency=low
 
+  [ Julian Andres Klode ]
   * data/templates/Debian.info.in: Squeeze will be 6.0, not 5.1
 
- -- Julian Andres Klode <jak@debian.org>  Mon, 15 Jun 2009 17:21:57 +0200
+  [ Stephan Peijnik ]
+  * apt/progress/__init__.py:
+    - add update_status_full() that takes file_size/partial_size as 
+      additional callback arguments
+    - add pulse_items() that takes a addtional "items" tuple that 
+      gives the user full access to the individual items that are
+      fetched
+  * python/progress.cc:
+    - low level code for update_status_full and pulse_items()
+    - better threading support
+  
+  [ Michael Vogt ]
+  * aptsources/distro.py:
+    - fix indent error that causes incorrect sources.list additons
+      (LP: #372224)
+  * python/progress.cc:
+    - fix crash in RunSimpleCallback()
+  * apt/cache.py:
+    - when the cache is run with a alternative rootdir, create
+      required dirs/files automatically
 
+ -- Michael Vogt <mvo@debian.org>  Mon, 20 Jul 2009 15:35:27 +0200
+  
 python-apt (0.7.10.4) unstable; urgency=low
 
   [ Michael Vogt ]
diff --git a/doc/examples/progress.py b/doc/examples/progress.py
index 2231001f..c007681f 100644
--- a/doc/examples/progress.py
+++ b/doc/examples/progress.py
@@ -35,15 +35,31 @@ class TextFetchProgress(apt.FetchProgress):
         pass
 
     def updateStatus(self, uri, descr, shortDescr, status):
-        print "UpdateStatus: '%s' '%s' '%s' '%i'" % (
+        print "UpdateStatus: '%s' '%s' '%s' '%i' " % (
             uri, descr, shortDescr, status)
 
+    def update_status_full(self, uri, descr, shortDescr, status, fileSize, 
+                           partialSize):
+        print "update_status_full: '%s' '%s' '%s' '%i' '%d/%d'" % (
+            uri, descr, shortDescr, status, partialSize, fileSize)
+
     def pulse(self):
         print "Pulse: CPS: %s/s; Bytes: %s/%s; Item: %s/%s" % (
             apt.SizeToStr(self.currentCPS), apt.SizeToStr(self.currentBytes),
             apt.SizeToStr(self.totalBytes), self.currentItems, self.totalItems)
         return True
 
+    def pulse_items(self, items):
+        print "Pulse: CPS: %s/s; Bytes: %s/%s; Item: %s/%s" % (
+            apt.SizeToStr(self.currentCPS), apt.SizeToStr(self.currentBytes),
+            apt.SizeToStr(self.totalBytes), self.currentItems, self.totalItems)
+        print "Pulse-Items: "
+        for itm in items:
+            uri, descr, shortDescr, fileSize, partialSize = itm
+            print " - '%s' '%s' '%s' '%d/%d'" % (
+                uri, descr, shortDescr, partialSize, fileSize)
+        return True
+
     def mediaChange(self, medium, drive):
         print "Please insert medium %s in drive %s" % (medium, drive)
         sys.stdin.readline()
diff --git a/po/python-apt.pot b/po/python-apt.pot
index 9c23c579..02120bfa 100644
--- a/po/python-apt.pot
+++ b/po/python-apt.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2009-04-12 19:07+0200\n"
+"POT-Creation-Date: 2009-07-03 12:55+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -359,11 +359,11 @@ msgstr ""
 msgid "Invalid unicode in description for '%s' (%s). Please report."
 msgstr ""
 
-#: ../apt/package.py:745 ../apt/package.py:849
+#: ../apt/package.py:747 ../apt/package.py:851
 msgid "The list of changes is not available"
 msgstr ""
 
-#: ../apt/package.py:853
+#: ../apt/package.py:855
 #, python-format
 msgid ""
 "The list of changes is not available yet.\n"
@@ -372,7 +372,7 @@ msgid ""
 "until the changes become available or try again later."
 msgstr ""
 
-#: ../apt/package.py:859
+#: ../apt/package.py:861
 msgid ""
 "Failed to download the list of changes. \n"
 "Please check your Internet connection."
diff --git a/python/generic.h b/python/generic.h
index ce79a54c..d2fcf42a 100644
--- a/python/generic.h
+++ b/python/generic.h
@@ -121,8 +121,9 @@ void CppOwnedDealloc(PyObject *iObj)
 {
    CppOwnedPyObject<T> *Obj = (CppOwnedPyObject<T> *)iObj;
    Obj->Object.~T();
-   if (Obj->Owner != 0)
+   if (Obj->Owner != 0) {
       Py_DECREF(Obj->Owner);
+   }
    PyObject_DEL(Obj);
 }
 
diff --git a/python/progress.cc b/python/progress.cc
index c5a1c138..94debe40 100644
--- a/python/progress.cc
+++ b/python/progress.cc
@@ -9,10 +9,13 @@
 #include <iostream>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <map>
+#include <utility>
 #include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
+#include "generic.h"
 #include "progress.h"
 
-
 // generic
 bool PyCallbackObj::RunSimpleCallback(const char* method_name,
 				      PyObject *arglist,
@@ -25,20 +28,25 @@ bool PyCallbackObj::RunSimpleCallback(const char* method_name,
 
    PyObject *method = PyObject_GetAttrString(callbackInst,(char*) method_name);
    if(method == NULL) {
-      // FIXME: make this silent
       //std::cerr << "Can't find '" << method_name << "' method" << std::endl;
       Py_XDECREF(arglist);
+      if (res) {
+	 Py_INCREF(Py_None);
+	 *res = Py_None;
+      }
       return false;
    }
    PyObject *result = PyEval_CallObject(method, arglist);
+   
    Py_XDECREF(arglist);
 
    if(result == NULL) {
       // exception happend
       std::cerr << "Error in function " << method_name << std::endl;
       PyErr_Print();
+      PyErr_Clear();
 
-      return NULL;
+      return false;
    }
    if(res != NULL)
       *res = result;
@@ -84,8 +92,10 @@ void PyOpProgress::Done()
 
 
 // apt interface
+
 bool PyFetchProgress::MediaChange(string Media, string Drive)
 {
+   PyCbObj_END_ALLOW_THREADS
    //std::cout << "MediaChange" << std::endl;
    PyObject *arglist = Py_BuildValue("(ss)", Media.c_str(), Drive.c_str());
    PyObject *result;
@@ -98,14 +108,35 @@ bool PyFetchProgress::MediaChange(string Media, string Drive)
    // FIXME: find out what it should return usually
    //std::cerr << "res is: " << res << std::endl;
 
+   PyCbObj_BEGIN_ALLOW_THREADS
    return res;
 }
 
 void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status)
 {
    //std::cout << "UpdateStatus: " << Itm.URI << " " << status << std::endl;
-   PyObject *arglist = Py_BuildValue("(sssi)", Itm.URI.c_str(), Itm.Description.c_str(), Itm.ShortDesc.c_str(), status);
+
+   // Added object file size and object partial size to
+   // parameters that are passed to updateStatus.
+   // -- Stephan
+   PyCbObj_END_ALLOW_THREADS
+   PyObject *arglist = Py_BuildValue("(sssikk)", Itm.URI.c_str(), 
+				     Itm.Description.c_str(), 
+				     Itm.ShortDesc.c_str(), 
+				     status,
+				     Itm.Owner->FileSize,
+				     Itm.Owner->PartialSize);
+
+   RunSimpleCallback("update_status_full", arglist);
+
+   // legacy version of the interface
+   arglist = Py_BuildValue("(sssi)", Itm.URI.c_str(), 
+				     Itm.Description.c_str(), 
+				     Itm.ShortDesc.c_str(), 
+                                     status);
    RunSimpleCallback("updateStatus", arglist);
+   PyCbObj_BEGIN_ALLOW_THREADS
+
 }
 
 void PyFetchProgress::IMSHit(pkgAcquire::ItemDesc &Itm)
@@ -141,12 +172,44 @@ void PyFetchProgress::Start()
 {
    //std::cout << "Start" << std::endl;
    pkgAcquireStatus::Start();
+
+   // These attributes should be initialized before the first callback (start)
+   // is invoked.
+   // -- Stephan
+   PyObject *o;
+
+   o = Py_BuildValue("f", 0.0f);
+   PyObject_SetAttrString(callbackInst, "currentCPS", o);
+   Py_XDECREF(o);
+   o = Py_BuildValue("f", 0.0f);
+   PyObject_SetAttrString(callbackInst, "currentBytes", o);
+   Py_XDECREF(o);
+   o = Py_BuildValue("i", 0);
+   PyObject_SetAttrString(callbackInst, "currentItems", o);
+   Py_XDECREF(o);
+   o = Py_BuildValue("i", 0);
+   PyObject_SetAttrString(callbackInst, "totalItems", o);
+   Py_XDECREF(o);
+   o = Py_BuildValue("f", 0.0f);
+   PyObject_SetAttrString(callbackInst, "totalBytes", o);
+   Py_XDECREF(o);
+
    RunSimpleCallback("start");
+   /* After calling the start method we can safely allow
+    * other Python threads to do their work for now.
+    */
+   PyCbObj_BEGIN_ALLOW_THREADS
 }
 
 
 void PyFetchProgress::Stop()
 {
+   /* After the stop operation occured no other threads
+    * are allowed. This is done so we have a matching 
+    * PyCbObj_END_ALLOW_THREADS to our previous
+    * PyCbObj_BEGIN_ALLOW_THREADS (Python requires this!).
+    */
+   PyCbObj_END_ALLOW_THREADS
    //std::cout << "Stop" << std::endl;
    pkgAcquireStatus::Stop();
    RunSimpleCallback("stop");
@@ -154,6 +217,7 @@ void PyFetchProgress::Stop()
 
 bool PyFetchProgress::Pulse(pkgAcquire * Owner)
 {
+   PyCbObj_END_ALLOW_THREADS
    pkgAcquireStatus::Pulse(Owner);
 
    //std::cout << "Pulse" << std::endl;
@@ -178,19 +242,90 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner)
    PyObject_SetAttrString(callbackInst, "totalBytes", o);
    Py_XDECREF(o);
 
-   PyObject *arglist = Py_BuildValue("()");
-   PyObject *result;
-   RunSimpleCallback("pulse", arglist, &result);
+   // Go through the list of items and add active items to the
+   // activeItems vector.
+   map<pkgAcquire::Worker *, pkgAcquire::ItemDesc *> activeItemMap;
+   
+   for(pkgAcquire::Worker *Worker = Owner->WorkersBegin();
+       Worker != 0; Worker = Owner->WorkerStep(Worker)) {
+
+     if (Worker->CurrentItem == 0) {
+       // Ignore workers with no item running
+       continue;
+     }
+     activeItemMap.insert(std::make_pair(Worker, Worker->CurrentItem));
+   }
+   
+   // Create the tuple that is passed as argument to pulse().
+   // This tuple contains activeItemMap.size() item tuples.
+   PyObject *arglist;
+
+   if (((int)activeItemMap.size()) > 0) {
+     PyObject *itemsTuple = PyTuple_New((Py_ssize_t) activeItemMap.size());
+
+     // Go through activeItems, create an item tuple in the form
+     // (URI, Description, ShortDesc, FileSize, PartialSize) and
+     // add that tuple to itemsTuple.
+     map<pkgAcquire::Worker *, pkgAcquire::ItemDesc *>::iterator iter;
+     int tuplePos;
+
+     for(tuplePos = 0, iter = activeItemMap.begin(); 
+	 iter != activeItemMap.end(); ++iter, tuplePos++) {
+       pkgAcquire::Worker *worker = iter->first;
+       pkgAcquire::ItemDesc *itm = iter->second;
+
+       PyObject *itmTuple = Py_BuildValue("(ssskk)", itm->URI.c_str(),
+					  itm->Description.c_str(),
+					  itm->ShortDesc.c_str(),
+					  worker->TotalSize,
+					  worker->CurrentSize);
+       PyTuple_SetItem(itemsTuple, tuplePos, itmTuple);
+     }
+
+     // Now our itemsTuple is ready for being passed to pulse().
+     // pulse() is going to receive a single argument, being the
+     // tuple of items, which again contains one tuple with item
+     // information per item.
+     //
+     // Python Example:
+     //
+     // class MyFetchProgress(FetchProgress):
+     //   def pulse(self, items):
+     //     for itm in items:
+     //       uri, desc, shortdesc, filesize, partialsize = itm
+     //
+     arglist = PyTuple_Pack(1, itemsTuple);
+   }
+   else {
+     arglist = Py_BuildValue("(())");
+   }
 
+   PyObject *result;
    bool res = true;
-   if(!PyArg_Parse(result, "b", &res))
+
+   RunSimpleCallback("pulse_items", arglist, &result);
+   if (result != NULL && PyArg_Parse(result, "b", &res) && res == false) {
+      // the user returned a explicit false here, stop
+      PyCbObj_BEGIN_ALLOW_THREADS
+      return false;
+   }
+
+   arglist = Py_BuildValue("()");
+   if (!RunSimpleCallback("pulse", arglist, &result)) {
+     PyCbObj_BEGIN_ALLOW_THREADS
+     return true;
+   }
+
+   if((result == NULL) || (!PyArg_Parse(result, "b", &res)))
    {
       // most of the time the user who subclasses the pulse() 
       // method forgot to add a return {True,False} so we just
       // assume he wants a True
+      PyCbObj_BEGIN_ALLOW_THREADS
       return true;
    }
 
+   PyCbObj_BEGIN_ALLOW_THREADS
    // fetching can be canceld by returning false
    return res;
 }
@@ -202,15 +337,19 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner)
 void PyInstallProgress::StartUpdate()
 {
    RunSimpleCallback("startUpdate");
+   PyCbObj_BEGIN_ALLOW_THREADS
 }
 
 void PyInstallProgress::UpdateInterface()
 {
+   PyCbObj_END_ALLOW_THREADS
    RunSimpleCallback("updateInterface");
+   PyCbObj_BEGIN_ALLOW_THREADS
 }
 
 void PyInstallProgress::FinishUpdate()
 {
+   PyCbObj_END_ALLOW_THREADS
    RunSimpleCallback("finishUpdate");
 }
 
@@ -269,9 +408,9 @@ pkgPackageManager::OrderResult PyInstallProgress::Run(pkgPackageManager *pm)
       _exit(res);
    }
 
-
    StartUpdate();
 
+   PyCbObj_END_ALLOW_THREADS
    if(PyObject_HasAttrString(callbackInst, "waitChild")) {
       PyObject *method = PyObject_GetAttrString(callbackInst, "waitChild");
       //std::cerr << "custom waitChild found" << std::endl;
@@ -286,14 +425,19 @@ pkgPackageManager::OrderResult PyInstallProgress::Run(pkgPackageManager *pm)
       int child_res;
       if(!PyArg_Parse(result, "i", &res) ) {
 	 std::cerr << "custom waitChild() result could not be parsed?"<< std::endl;
+	 PyCbObj_BEGIN_ALLOW_THREADS
 	 return pkgPackageManager::Failed;
       }
+      PyCbObj_BEGIN_ALLOW_THREADS
       //std::cerr << "got child_res: " << res << std::endl;
    } else {
       //std::cerr << "using build-in waitpid()" << std::endl;
-
-      while (waitpid(child_id, &ret, WNOHANG) == 0)
+      PyCbObj_BEGIN_ALLOW_THREADS
+      while (waitpid(child_id, &ret, WNOHANG) == 0) {
+	 PyCbObj_END_ALLOW_THREADS
 	 UpdateInterface();
+         PyCbObj_BEGIN_ALLOW_THREADS
+      }
 
       res = (pkgPackageManager::OrderResult) WEXITSTATUS(ret);
       //std::cerr << "build-in waitpid() got: " << res << std::endl;
diff --git a/python/progress.h b/python/progress.h
index 5ac67b1c..29243bfc 100644
--- a/python/progress.h
+++ b/python/progress.h
@@ -15,10 +15,27 @@
 #include <apt-pkg/cdrom.h>
 #include <Python.h>
 
+/* PyCbObj_BEGIN_ALLOW_THREADS and PyCbObj_END_ALLOW_THREADS are sligthly
+ * modified versions of Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS.
+ * Instead of storing the thread state in a function-local variable these
+ * use a class attribute (with the same) name, allowing blocking and 
+ * unblocking from different class methods.
+ * Py_BLOCK_THREADS and Py_UNBLOCK_THREADS do not define their own
+ * local variable but use the one provided by PyCbObj_BEGIN_ALLOW_THREADS
+ * and thus are the same as Py_BLOCK_THREADS and Py_UNBLOCK_THREADS.
+ */
+#define PyCbObj_BEGIN_ALLOW_THREADS \
+  _save = PyEval_SaveThread();
+#define PyCbObj_END_ALLOW_THREADS \
+  PyEval_RestoreThread(_save); \
+  _save = NULL;
+#define PyCbObj_BLOCK_THREADS Py_BLOCK_THREADS
+#define PyCbObj_UNBLOCK_THREADS Py_UNBLOCK_THREADS
 
 class PyCallbackObj {
  protected:
    PyObject *callbackInst;
+   PyThreadState *_save;
 
  public:
    void setCallbackInst(PyObject *o) {