From c639dfe4d9c77bf7406bb6e4b433c297d293f709 Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Mon, 4 May 2009 15:39:18 +0200 Subject: More information available for fetch progress callbacks. Better exception handling (we really shouldn't segfault on an exception). --- apt/progress/__init__.py | 5 ++- doc/examples/progress.py | 14 ++++-- po/python-apt.pot | 8 ++-- python/progress.cc | 108 ++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 119 insertions(+), 16 deletions(-) diff --git a/apt/progress/__init__.py b/apt/progress/__init__.py index 769942ce..68441cfa 100644 --- a/apt/progress/__init__.py +++ b/apt/progress/__init__.py @@ -107,13 +107,14 @@ class FetchProgress(object): def stop(self): """Called when all files have been fetched.""" - def updateStatus(self, uri, descr, shortDescr, status): + def updateStatus(self, uri, descr, shortDescr, status, fileSize, + partialSize): """Called when the status of an item changes. This happens eg. when the downloads fails or is completed. """ - def pulse(self): + def pulse(self, items): """Called periodically to update the user interface. Return True to continue or False to cancel. diff --git a/doc/examples/progress.py b/doc/examples/progress.py index 2231001f..f7650f69 100644 --- a/doc/examples/progress.py +++ b/doc/examples/progress.py @@ -34,14 +34,20 @@ class TextFetchProgress(apt.FetchProgress): def stop(self): pass - def updateStatus(self, uri, descr, shortDescr, status): - print "UpdateStatus: '%s' '%s' '%s' '%i'" % ( - uri, descr, shortDescr, status) + def updateStatus(self, uri, descr, shortDescr, status, fileSize, + partialSize): + print "UpdateStatus: '%s' '%s' '%s' '%i' '%d/%d'" % ( + uri, descr, shortDescr, status, partialSize, fileSize) - def pulse(self): + def pulse(self, items): print "Pulse: CPS: %s/s; Bytes: %s/%s; Item: %s/%s" % ( apt.SizeToStr(self.currentCPS), apt.SizeToStr(self.currentBytes), apt.SizeToStr(self.totalBytes), self.currentItems, self.totalItems) + print "Pulse-Items: " + for itm in items: + uri, descr, shortDescr, fileSize, partialSize = itm + print " - '%s' '%s' '%s' '%d/%d'" % ( + uri, descr, shortDescr, partialSize, fileSize return True def mediaChange(self, medium, drive): diff --git a/po/python-apt.pot b/po/python-apt.pot index 9c23c579..27f7e1af 100644 --- a/po/python-apt.pot +++ b/po/python-apt.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2009-04-12 19:07+0200\n" +"POT-Creation-Date: 2009-04-30 12:52+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -359,11 +359,11 @@ msgstr "" msgid "Invalid unicode in description for '%s' (%s). Please report." msgstr "" -#: ../apt/package.py:745 ../apt/package.py:849 +#: ../apt/package.py:747 ../apt/package.py:851 msgid "The list of changes is not available" msgstr "" -#: ../apt/package.py:853 +#: ../apt/package.py:855 #, python-format msgid "" "The list of changes is not available yet.\n" @@ -372,7 +372,7 @@ msgid "" "until the changes become available or try again later." msgstr "" -#: ../apt/package.py:859 +#: ../apt/package.py:861 msgid "" "Failed to download the list of changes. \n" "Please check your Internet connection." diff --git a/python/progress.cc b/python/progress.cc index c5a1c138..a30ea2e7 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -9,10 +9,12 @@ #include #include #include +#include +#include #include +#include #include "progress.h" - // generic bool PyCallbackObj::RunSimpleCallback(const char* method_name, PyObject *arglist, @@ -31,14 +33,16 @@ bool PyCallbackObj::RunSimpleCallback(const char* method_name, return false; } PyObject *result = PyEval_CallObject(method, arglist); + Py_XDECREF(arglist); if(result == NULL) { // exception happend std::cerr << "Error in function " << method_name << std::endl; PyErr_Print(); + PyErr_Clear(); - return NULL; + return false; } if(res != NULL) *res = result; @@ -84,6 +88,7 @@ void PyOpProgress::Done() // apt interface + bool PyFetchProgress::MediaChange(string Media, string Drive) { //std::cout << "MediaChange" << std::endl; @@ -104,7 +109,17 @@ bool PyFetchProgress::MediaChange(string Media, string Drive) void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) { //std::cout << "UpdateStatus: " << Itm.URI << " " << status << std::endl; - PyObject *arglist = Py_BuildValue("(sssi)", Itm.URI.c_str(), Itm.Description.c_str(), Itm.ShortDesc.c_str(), status); + + // Added object file size and object partial size to + // parameters that are passed to updateStatus. + // -- Stephan + PyObject *arglist = Py_BuildValue("(sssikk)", Itm.URI.c_str(), + Itm.Description.c_str(), + Itm.ShortDesc.c_str(), + status, + Itm.Owner->FileSize, + Itm.Owner->PartialSize); + RunSimpleCallback("updateStatus", arglist); } @@ -141,6 +156,28 @@ void PyFetchProgress::Start() { //std::cout << "Start" << std::endl; pkgAcquireStatus::Start(); + + // These attributes should be initialized before the first callback (start) + // is invoked. + // -- Stephan + PyObject *o; + + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "currentCPS", o); + Py_XDECREF(o); + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "currentBytes", o); + Py_XDECREF(o); + o = Py_BuildValue("i", 0); + PyObject_SetAttrString(callbackInst, "currentItems", o); + Py_XDECREF(o); + o = Py_BuildValue("i", 0); + PyObject_SetAttrString(callbackInst, "totalItems", o); + Py_XDECREF(o); + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "totalBytes", o); + Py_XDECREF(o); + RunSimpleCallback("start"); } @@ -178,12 +215,71 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) PyObject_SetAttrString(callbackInst, "totalBytes", o); Py_XDECREF(o); - PyObject *arglist = Py_BuildValue("()"); + // Go through the list of items and add active items to the + // activeItems vector. + map activeItemMap; + + for(pkgAcquire::Worker *Worker = Owner->WorkersBegin(); + Worker != 0; Worker = Owner->WorkerStep(Worker)) { + + if (Worker->CurrentItem == 0) { + // Ignore workers with no item running + continue; + } + activeItemMap.insert(std::make_pair(Worker, Worker->CurrentItem)); + } + + // Create the tuple that is passed as argument to pulse(). + // This tuple contains activeItemMap.size() item tuples. + PyObject *arglist; + + if (((int)activeItemMap.size()) > 0) { + PyObject *itemsTuple = PyTuple_New((Py_ssize_t) activeItemMap.size()); + + // Go through activeItems, create an item tuple in the form + // (URI, Description, ShortDesc, FileSize, PartialSize) and + // add that tuple to itemsTuple. + map::iterator iter; + int tuplePos; + + for(tuplePos = 0, iter = activeItemMap.begin(); + iter != activeItemMap.end(); ++iter, tuplePos++) { + pkgAcquire::Worker *worker = iter->first; + pkgAcquire::ItemDesc *itm = iter->second; + + PyObject *itmTuple = Py_BuildValue("(ssskk)", itm->URI.c_str(), + itm->Description.c_str(), + itm->ShortDesc.c_str(), + worker->TotalSize, + worker->CurrentSize); + PyTuple_SetItem(itemsTuple, tuplePos, itmTuple); + } + + // Now our itemsTuple is ready for being passed to pulse(). + // pulse() is going to receive a single argument, being the + // tuple of items, which again contains one tuple with item + // information per item. + // + // Python Example: + // + // class MyFetchProgress(FetchProgress): + // def pulse(self, items): + // for itm in items: + // uri, desc, shortdesc, filesize, partialsize = itm + // + arglist = PyTuple_Pack(1, itemsTuple); + } + else { + arglist = Py_BuildValue("(())"); + } + PyObject *result; - RunSimpleCallback("pulse", arglist, &result); + if (!RunSimpleCallback("pulse", arglist, &result)) { + return true; + } bool res = true; - if(!PyArg_Parse(result, "b", &res)) + if((result == NULL) || (!PyArg_Parse(result, "b", &res))) { // most of the time the user who subclasses the pulse() // method forgot to add a return {True,False} so we just -- cgit v1.2.3 From 1928a8fab432b219c6286ff08b142c57c6df5e6b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 15 Jun 2009 10:56:01 +0200 Subject: * python/progress.cc: - fix crash in RunSimpleCallback() * apt/cache.py: - when the cache is run with a alternative rootdir, create required dirs/files automatically --- apt/cache.py | 25 +++++++++++++++++++++++++ debian/changelog | 5 +++++ python/progress.cc | 5 ++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/apt/cache.py b/apt/cache.py index cc425ccb..94a77fd8 100644 --- a/apt/cache.py +++ b/apt/cache.py @@ -60,8 +60,33 @@ class Cache(object): apt_pkg.Config.Set("Dir", rootdir) apt_pkg.Config.Set("Dir::State::status", rootdir + "/var/lib/dpkg/status") + # create required dirs/files when run with special rootdir + # automatically + self._check_and_create_required_dirs(rootdir) self.open(progress) + def _check_and_create_required_dirs(self, rootdir): + """ + check if the required apt directories/files are there and if + not create them + """ + files = ["/var/lib/dpkg/status", + "/etc/apt/sources.list", + ] + dirs = ["/var/lib/dpkg", + "/etc/apt/", + "/var/cache/apt/archives/partial", + "/var/lib/apt/lists/partial", + ] + for d in dirs: + if not os.path.exists(rootdir+d): + print "creating: ",rootdir+d + os.makedirs(rootdir+d) + for f in files: + if not os.path.exists(rootdir+f): + open(rootdir+f,"w") + + def _runCallbacks(self, name): """ internal helper to run a callback """ if name in self._callbacks: diff --git a/debian/changelog b/debian/changelog index a3faab79..0f9463b2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -99,8 +99,13 @@ python-apt (0.7.9) unstable; urgency=low closes: #497049 * apt/package.py: - avoid uneeded interal references in the Package objects + * python/progress.cc: + - fix crash in RunSimpleCallback() * aptsources/sourceslist.py: - fix bug in invalid lines detection (LP: #324614) + * apt/cache.py: + - when the cache is run with a alternative rootdir, create + required dirs/files automatically -- Michael Vogt Thu, 19 Mar 2009 13:39:21 +0100 diff --git a/python/progress.cc b/python/progress.cc index 14948d3c..95cece3b 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -27,9 +27,12 @@ bool PyCallbackObj::RunSimpleCallback(const char* method_name, PyObject *method = PyObject_GetAttrString(callbackInst,(char*) method_name); if(method == NULL) { - // FIXME: make this silent //std::cerr << "Can't find '" << method_name << "' method" << std::endl; Py_XDECREF(arglist); + if (res) { + Py_INCREF(Py_None); + *res = Py_None; + } return false; } PyObject *result = PyEval_CallObject(method, arglist); -- cgit v1.2.3 From 87f6d28c1555ee70da9dddb9625cea0c4d6ab9a5 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 15 Jun 2009 17:22:49 +0200 Subject: data/templates/Debian.info.in: Squeeze will be 6.0, not 5.1 --- data/templates/Debian.info.in | 2 +- debian/changelog | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/data/templates/Debian.info.in b/data/templates/Debian.info.in index e80f0f6c..e5a1b424 100644 --- a/data/templates/Debian.info.in +++ b/data/templates/Debian.info.in @@ -5,7 +5,7 @@ RepositoryType: deb BaseURI: http://http.us.debian.org/debian/ MatchURI: ftp[0-9]*\.[a-z]\.debian\.org MirrorsFile: /usr/share/python-apt/templates/Debian.mirrors -_Description: Debian 5.1 'Squeeze' +_Description: Debian 6.0 'Squeeze' Component: main _CompDescription: Officially supported Component: contrib diff --git a/debian/changelog b/debian/changelog index 10d40aba..b3b811c2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-apt (0.7.10.5) UNRELEASED; urgency=low + + * data/templates/Debian.info.in: Squeeze will be 6.0, not 5.1 + + -- Julian Andres Klode Mon, 15 Jun 2009 17:21:57 +0200 + python-apt (0.7.10.4) unstable; urgency=low [ Michael Vogt ] -- cgit v1.2.3 From d8c5f345039c9577d7326670afeeb7666c8d9c7c Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Thu, 2 Jul 2009 12:41:04 +0200 Subject: C code must allow other python threads to work every now and then in progress.cc (take one). --- python/progress.cc | 20 ++++++++++++++++++++ python/progress.h | 17 +++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/python/progress.cc b/python/progress.cc index 95cece3b..ab0deaef 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -94,6 +94,7 @@ void PyOpProgress::Done() bool PyFetchProgress::MediaChange(string Media, string Drive) { + PyCbObj_END_ALLOW_THREADS //std::cout << "MediaChange" << std::endl; PyObject *arglist = Py_BuildValue("(ss)", Media.c_str(), Drive.c_str()); PyObject *result; @@ -106,6 +107,7 @@ bool PyFetchProgress::MediaChange(string Media, string Drive) // FIXME: find out what it should return usually //std::cerr << "res is: " << res << std::endl; + PyCbObj_BEGIN_ALLOW_THREADS return res; } @@ -116,6 +118,7 @@ void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) // Added object file size and object partial size to // parameters that are passed to updateStatus. // -- Stephan + PyCbObj_END_ALLOW_THREADS PyObject *arglist = Py_BuildValue("(sssikk)", Itm.URI.c_str(), Itm.Description.c_str(), Itm.ShortDesc.c_str(), @@ -131,6 +134,7 @@ void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) Itm.ShortDesc.c_str(), status); RunSimpleCallback("updateStatus", arglist); + PyCbObj_BEGIN_ALLOW_THREADS } @@ -190,11 +194,22 @@ void PyFetchProgress::Start() Py_XDECREF(o); RunSimpleCallback("start"); + /* After calling the start method we can safely allow + * other Python threads to do their work for now. + */ + PyCbObj_BEGIN_ALLOW_THREADS } void PyFetchProgress::Stop() { + /* After the stop operation occured no other threads + * are allowed. This is done so we have a matching + * PyCbObj_END_ALLOW_THREADS to our previous + * PyCbObj_BEGIN_ALLOW_THREADS (Python requires this!). + */ + std::cout << "PyFetchProgress::STOP" << std::endl; + PyCbObj_END_ALLOW_THREADS //std::cout << "Stop" << std::endl; pkgAcquireStatus::Stop(); RunSimpleCallback("stop"); @@ -202,6 +217,7 @@ void PyFetchProgress::Stop() bool PyFetchProgress::Pulse(pkgAcquire * Owner) { + PyCbObj_END_ALLOW_THREADS pkgAcquireStatus::Pulse(Owner); //std::cout << "Pulse" << std::endl; @@ -290,11 +306,13 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) RunSimpleCallback("pulse_items", arglist, &result); if (result != NULL && PyArg_Parse(result, "b", &res) && res == false) { // the user returned a explicit false here, stop + PyCbObj_BEGIN_ALLOW_THREADS return false; } arglist = Py_BuildValue("()"); if (!RunSimpleCallback("pulse", arglist, &result)) { + PyCbObj_BEGIN_ALLOW_THREADS return true; } @@ -303,9 +321,11 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) // most of the time the user who subclasses the pulse() // method forgot to add a return {True,False} so we just // assume he wants a True + PyCbObj_BEGIN_ALLOW_THREADS return true; } + PyCbObj_BEGIN_ALLOW_THREADS // fetching can be canceld by returning false return res; } diff --git a/python/progress.h b/python/progress.h index 5ac67b1c..29243bfc 100644 --- a/python/progress.h +++ b/python/progress.h @@ -15,10 +15,27 @@ #include #include +/* PyCbObj_BEGIN_ALLOW_THREADS and PyCbObj_END_ALLOW_THREADS are sligthly + * modified versions of Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS. + * Instead of storing the thread state in a function-local variable these + * use a class attribute (with the same) name, allowing blocking and + * unblocking from different class methods. + * Py_BLOCK_THREADS and Py_UNBLOCK_THREADS do not define their own + * local variable but use the one provided by PyCbObj_BEGIN_ALLOW_THREADS + * and thus are the same as Py_BLOCK_THREADS and Py_UNBLOCK_THREADS. + */ +#define PyCbObj_BEGIN_ALLOW_THREADS \ + _save = PyEval_SaveThread(); +#define PyCbObj_END_ALLOW_THREADS \ + PyEval_RestoreThread(_save); \ + _save = NULL; +#define PyCbObj_BLOCK_THREADS Py_BLOCK_THREADS +#define PyCbObj_UNBLOCK_THREADS Py_UNBLOCK_THREADS class PyCallbackObj { protected: PyObject *callbackInst; + PyThreadState *_save; public: void setCallbackInst(PyObject *o) { -- cgit v1.2.3 From 4882fc138b9439469bb934f2db659bc97f131725 Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Thu, 2 Jul 2009 12:42:39 +0200 Subject: Removed debug print statement. --- python/progress.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/python/progress.cc b/python/progress.cc index ab0deaef..027a51b4 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -208,7 +208,6 @@ void PyFetchProgress::Stop() * PyCbObj_END_ALLOW_THREADS to our previous * PyCbObj_BEGIN_ALLOW_THREADS (Python requires this!). */ - std::cout << "PyFetchProgress::STOP" << std::endl; PyCbObj_END_ALLOW_THREADS //std::cout << "Stop" << std::endl; pkgAcquireStatus::Stop(); -- cgit v1.2.3 From d28170024ef4d2f01fd9096a3ec785cf424c0846 Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Tue, 7 Jul 2009 18:44:10 +0200 Subject: Removed separate way of defining Py_ssize_t in progress.h. Now using method present in generic.h in progress.cc. --- debian/changelog | 2 +- python/progress.cc | 1 + python/progress.h | 6 ------ 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index db6204ba..ce96cc9e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-apt (0.7.10.4) UNRELEASED; urgency=low +python-apt (0.7.11.0) UNRELEASED; urgency=low [ Stephan Peijnik ] * apt/progress/__init__.py: diff --git a/python/progress.cc b/python/progress.cc index cbb49671..94debe40 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -13,6 +13,7 @@ #include #include #include +#include "generic.h" #include "progress.h" // generic diff --git a/python/progress.h b/python/progress.h index 659eb113..29243bfc 100644 --- a/python/progress.h +++ b/python/progress.h @@ -15,12 +15,6 @@ #include #include -/* Python 2.4 compatibility */ -#if (PY_VERSION_HEX < 0x02050000) -typedef int Py_ssize_t; -#endif - - /* PyCbObj_BEGIN_ALLOW_THREADS and PyCbObj_END_ALLOW_THREADS are sligthly * modified versions of Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS. * Instead of storing the thread state in a function-local variable these -- cgit v1.2.3 From d445714639596ac57d4dfeb44eb6ca69fbea8190 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 13 Jul 2009 14:51:21 +0200 Subject: * aptsources/distro.py: - fix indent error that causes incorrect sources.list additons (LP: #372224) --- aptsources/distro.py | 4 ++-- debian/changelog | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/aptsources/distro.py b/aptsources/distro.py index bbb8ba50..5398d4a3 100644 --- a/aptsources/distro.py +++ b/aptsources/distro.py @@ -319,12 +319,12 @@ class Distribution: if s.type == self.binary_type: if s.dist not in comps_per_dist: comps_per_dist[s.dist] = set() - map(comps_per_dist[s.dist].add, s.comps) + map(comps_per_dist[s.dist].add, s.comps) for s in self.source_code_sources: if s.type == self.source_type: if s.dist not in comps_per_sdist: comps_per_sdist[s.dist] = set() - map(comps_per_sdist[s.dist].add, s.comps) + map(comps_per_sdist[s.dist].add, s.comps) # check if there is a main source at all if len(self.main_sources) < 1: diff --git a/debian/changelog b/debian/changelog index db6204ba..cb1c932e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -10,6 +10,11 @@ python-apt (0.7.10.4) UNRELEASED; urgency=low * python/progress.cc: - low level code for update_status_full and pulse_items() - better threading support + + [ Michael Vogt ] + * aptsources/distro.py: + - fix indent error that causes incorrect sources.list additons + (LP: #372224) -- Michael Vogt Tue, 05 May 2009 11:57:57 +0200 -- cgit v1.2.3