From 1b5b026c1013dc20b572a7fd2a690fe451248ab0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 5 May 2009 11:59:53 +0200 Subject: * apt/progress/__init__.py: - add update_status_full() that takes file_size/partial_size as additional callback arguments - add pulse_items() that takes a addtional "items" tuple that gives the user full access to the individual items that are fetched * python/progress.cc: - low level code for update_status_full and pulse_items() --- python/progress.cc | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) (limited to 'python') diff --git a/python/progress.cc b/python/progress.cc index c5a1c138..14948d3c 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -9,10 +9,12 @@ #include #include #include +#include +#include #include +#include #include "progress.h" - // generic bool PyCallbackObj::RunSimpleCallback(const char* method_name, PyObject *arglist, @@ -31,14 +33,16 @@ bool PyCallbackObj::RunSimpleCallback(const char* method_name, return false; } PyObject *result = PyEval_CallObject(method, arglist); + Py_XDECREF(arglist); if(result == NULL) { // exception happend std::cerr << "Error in function " << method_name << std::endl; PyErr_Print(); + PyErr_Clear(); - return NULL; + return false; } if(res != NULL) *res = result; @@ -84,6 +88,7 @@ void PyOpProgress::Done() // apt interface + bool PyFetchProgress::MediaChange(string Media, string Drive) { //std::cout << "MediaChange" << std::endl; @@ -104,8 +109,26 @@ bool PyFetchProgress::MediaChange(string Media, string Drive) void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) { //std::cout << "UpdateStatus: " << Itm.URI << " " << status << std::endl; - PyObject *arglist = Py_BuildValue("(sssi)", Itm.URI.c_str(), Itm.Description.c_str(), Itm.ShortDesc.c_str(), status); + + // Added object file size and object partial size to + // parameters that are passed to updateStatus. + // -- Stephan + PyObject *arglist = Py_BuildValue("(sssikk)", Itm.URI.c_str(), + Itm.Description.c_str(), + Itm.ShortDesc.c_str(), + status, + Itm.Owner->FileSize, + Itm.Owner->PartialSize); + + RunSimpleCallback("update_status_full", arglist); + + // legacy version of the interface + arglist = Py_BuildValue("(sssi)", Itm.URI.c_str(), + Itm.Description.c_str(), + Itm.ShortDesc.c_str(), + status); RunSimpleCallback("updateStatus", arglist); + } void PyFetchProgress::IMSHit(pkgAcquire::ItemDesc &Itm) @@ -141,6 +164,28 @@ void PyFetchProgress::Start() { //std::cout << "Start" << std::endl; pkgAcquireStatus::Start(); + + // These attributes should be initialized before the first callback (start) + // is invoked. + // -- Stephan + PyObject *o; + + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "currentCPS", o); + Py_XDECREF(o); + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "currentBytes", o); + Py_XDECREF(o); + o = Py_BuildValue("i", 0); + PyObject_SetAttrString(callbackInst, "currentItems", o); + Py_XDECREF(o); + o = Py_BuildValue("i", 0); + PyObject_SetAttrString(callbackInst, "totalItems", o); + Py_XDECREF(o); + o = Py_BuildValue("f", 0.0f); + PyObject_SetAttrString(callbackInst, "totalBytes", o); + Py_XDECREF(o); + RunSimpleCallback("start"); } @@ -178,12 +223,79 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) PyObject_SetAttrString(callbackInst, "totalBytes", o); Py_XDECREF(o); - PyObject *arglist = Py_BuildValue("()"); - PyObject *result; - RunSimpleCallback("pulse", arglist, &result); + // Go through the list of items and add active items to the + // activeItems vector. + map activeItemMap; + + for(pkgAcquire::Worker *Worker = Owner->WorkersBegin(); + Worker != 0; Worker = Owner->WorkerStep(Worker)) { + + if (Worker->CurrentItem == 0) { + // Ignore workers with no item running + continue; + } + activeItemMap.insert(std::make_pair(Worker, Worker->CurrentItem)); + } + + // Create the tuple that is passed as argument to pulse(). + // This tuple contains activeItemMap.size() item tuples. + PyObject *arglist; + + if (((int)activeItemMap.size()) > 0) { + PyObject *itemsTuple = PyTuple_New((Py_ssize_t) activeItemMap.size()); + + // Go through activeItems, create an item tuple in the form + // (URI, Description, ShortDesc, FileSize, PartialSize) and + // add that tuple to itemsTuple. + map::iterator iter; + int tuplePos; + + for(tuplePos = 0, iter = activeItemMap.begin(); + iter != activeItemMap.end(); ++iter, tuplePos++) { + pkgAcquire::Worker *worker = iter->first; + pkgAcquire::ItemDesc *itm = iter->second; + + PyObject *itmTuple = Py_BuildValue("(ssskk)", itm->URI.c_str(), + itm->Description.c_str(), + itm->ShortDesc.c_str(), + worker->TotalSize, + worker->CurrentSize); + PyTuple_SetItem(itemsTuple, tuplePos, itmTuple); + } + + // Now our itemsTuple is ready for being passed to pulse(). + // pulse() is going to receive a single argument, being the + // tuple of items, which again contains one tuple with item + // information per item. + // + // Python Example: + // + // class MyFetchProgress(FetchProgress): + // def pulse(self, items): + // for itm in items: + // uri, desc, shortdesc, filesize, partialsize = itm + // + arglist = PyTuple_Pack(1, itemsTuple); + } + else { + arglist = Py_BuildValue("(())"); + } + PyObject *result; bool res = true; - if(!PyArg_Parse(result, "b", &res)) + + RunSimpleCallback("pulse_items", arglist, &result); + if (result != NULL && PyArg_Parse(result, "b", &res) && res == false) { + // the user returned a explicit false here, stop + return false; + } + + arglist = Py_BuildValue("()"); + if (!RunSimpleCallback("pulse", arglist, &result)) { + return true; + } + + if((result == NULL) || (!PyArg_Parse(result, "b", &res))) { // most of the time the user who subclasses the pulse() // method forgot to add a return {True,False} so we just -- cgit v1.2.3 From 1928a8fab432b219c6286ff08b142c57c6df5e6b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 15 Jun 2009 10:56:01 +0200 Subject: * python/progress.cc: - fix crash in RunSimpleCallback() * apt/cache.py: - when the cache is run with a alternative rootdir, create required dirs/files automatically --- apt/cache.py | 25 +++++++++++++++++++++++++ debian/changelog | 5 +++++ python/progress.cc | 5 ++++- 3 files changed, 34 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/apt/cache.py b/apt/cache.py index cc425ccb..94a77fd8 100644 --- a/apt/cache.py +++ b/apt/cache.py @@ -60,8 +60,33 @@ class Cache(object): apt_pkg.Config.Set("Dir", rootdir) apt_pkg.Config.Set("Dir::State::status", rootdir + "/var/lib/dpkg/status") + # create required dirs/files when run with special rootdir + # automatically + self._check_and_create_required_dirs(rootdir) self.open(progress) + def _check_and_create_required_dirs(self, rootdir): + """ + check if the required apt directories/files are there and if + not create them + """ + files = ["/var/lib/dpkg/status", + "/etc/apt/sources.list", + ] + dirs = ["/var/lib/dpkg", + "/etc/apt/", + "/var/cache/apt/archives/partial", + "/var/lib/apt/lists/partial", + ] + for d in dirs: + if not os.path.exists(rootdir+d): + print "creating: ",rootdir+d + os.makedirs(rootdir+d) + for f in files: + if not os.path.exists(rootdir+f): + open(rootdir+f,"w") + + def _runCallbacks(self, name): """ internal helper to run a callback """ if name in self._callbacks: diff --git a/debian/changelog b/debian/changelog index a3faab79..0f9463b2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -99,8 +99,13 @@ python-apt (0.7.9) unstable; urgency=low closes: #497049 * apt/package.py: - avoid uneeded interal references in the Package objects + * python/progress.cc: + - fix crash in RunSimpleCallback() * aptsources/sourceslist.py: - fix bug in invalid lines detection (LP: #324614) + * apt/cache.py: + - when the cache is run with a alternative rootdir, create + required dirs/files automatically -- Michael Vogt Thu, 19 Mar 2009 13:39:21 +0100 diff --git a/python/progress.cc b/python/progress.cc index 14948d3c..95cece3b 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -27,9 +27,12 @@ bool PyCallbackObj::RunSimpleCallback(const char* method_name, PyObject *method = PyObject_GetAttrString(callbackInst,(char*) method_name); if(method == NULL) { - // FIXME: make this silent //std::cerr << "Can't find '" << method_name << "' method" << std::endl; Py_XDECREF(arglist); + if (res) { + Py_INCREF(Py_None); + *res = Py_None; + } return false; } PyObject *result = PyEval_CallObject(method, arglist); -- cgit v1.2.3 From d8c5f345039c9577d7326670afeeb7666c8d9c7c Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Thu, 2 Jul 2009 12:41:04 +0200 Subject: C code must allow other python threads to work every now and then in progress.cc (take one). --- python/progress.cc | 20 ++++++++++++++++++++ python/progress.h | 17 +++++++++++++++++ 2 files changed, 37 insertions(+) (limited to 'python') diff --git a/python/progress.cc b/python/progress.cc index 95cece3b..ab0deaef 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -94,6 +94,7 @@ void PyOpProgress::Done() bool PyFetchProgress::MediaChange(string Media, string Drive) { + PyCbObj_END_ALLOW_THREADS //std::cout << "MediaChange" << std::endl; PyObject *arglist = Py_BuildValue("(ss)", Media.c_str(), Drive.c_str()); PyObject *result; @@ -106,6 +107,7 @@ bool PyFetchProgress::MediaChange(string Media, string Drive) // FIXME: find out what it should return usually //std::cerr << "res is: " << res << std::endl; + PyCbObj_BEGIN_ALLOW_THREADS return res; } @@ -116,6 +118,7 @@ void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) // Added object file size and object partial size to // parameters that are passed to updateStatus. // -- Stephan + PyCbObj_END_ALLOW_THREADS PyObject *arglist = Py_BuildValue("(sssikk)", Itm.URI.c_str(), Itm.Description.c_str(), Itm.ShortDesc.c_str(), @@ -131,6 +134,7 @@ void PyFetchProgress::UpdateStatus(pkgAcquire::ItemDesc &Itm, int status) Itm.ShortDesc.c_str(), status); RunSimpleCallback("updateStatus", arglist); + PyCbObj_BEGIN_ALLOW_THREADS } @@ -190,11 +194,22 @@ void PyFetchProgress::Start() Py_XDECREF(o); RunSimpleCallback("start"); + /* After calling the start method we can safely allow + * other Python threads to do their work for now. + */ + PyCbObj_BEGIN_ALLOW_THREADS } void PyFetchProgress::Stop() { + /* After the stop operation occured no other threads + * are allowed. This is done so we have a matching + * PyCbObj_END_ALLOW_THREADS to our previous + * PyCbObj_BEGIN_ALLOW_THREADS (Python requires this!). + */ + std::cout << "PyFetchProgress::STOP" << std::endl; + PyCbObj_END_ALLOW_THREADS //std::cout << "Stop" << std::endl; pkgAcquireStatus::Stop(); RunSimpleCallback("stop"); @@ -202,6 +217,7 @@ void PyFetchProgress::Stop() bool PyFetchProgress::Pulse(pkgAcquire * Owner) { + PyCbObj_END_ALLOW_THREADS pkgAcquireStatus::Pulse(Owner); //std::cout << "Pulse" << std::endl; @@ -290,11 +306,13 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) RunSimpleCallback("pulse_items", arglist, &result); if (result != NULL && PyArg_Parse(result, "b", &res) && res == false) { // the user returned a explicit false here, stop + PyCbObj_BEGIN_ALLOW_THREADS return false; } arglist = Py_BuildValue("()"); if (!RunSimpleCallback("pulse", arglist, &result)) { + PyCbObj_BEGIN_ALLOW_THREADS return true; } @@ -303,9 +321,11 @@ bool PyFetchProgress::Pulse(pkgAcquire * Owner) // most of the time the user who subclasses the pulse() // method forgot to add a return {True,False} so we just // assume he wants a True + PyCbObj_BEGIN_ALLOW_THREADS return true; } + PyCbObj_BEGIN_ALLOW_THREADS // fetching can be canceld by returning false return res; } diff --git a/python/progress.h b/python/progress.h index 5ac67b1c..29243bfc 100644 --- a/python/progress.h +++ b/python/progress.h @@ -15,10 +15,27 @@ #include #include +/* PyCbObj_BEGIN_ALLOW_THREADS and PyCbObj_END_ALLOW_THREADS are sligthly + * modified versions of Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS. + * Instead of storing the thread state in a function-local variable these + * use a class attribute (with the same) name, allowing blocking and + * unblocking from different class methods. + * Py_BLOCK_THREADS and Py_UNBLOCK_THREADS do not define their own + * local variable but use the one provided by PyCbObj_BEGIN_ALLOW_THREADS + * and thus are the same as Py_BLOCK_THREADS and Py_UNBLOCK_THREADS. + */ +#define PyCbObj_BEGIN_ALLOW_THREADS \ + _save = PyEval_SaveThread(); +#define PyCbObj_END_ALLOW_THREADS \ + PyEval_RestoreThread(_save); \ + _save = NULL; +#define PyCbObj_BLOCK_THREADS Py_BLOCK_THREADS +#define PyCbObj_UNBLOCK_THREADS Py_UNBLOCK_THREADS class PyCallbackObj { protected: PyObject *callbackInst; + PyThreadState *_save; public: void setCallbackInst(PyObject *o) { -- cgit v1.2.3 From 4882fc138b9439469bb934f2db659bc97f131725 Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Thu, 2 Jul 2009 12:42:39 +0200 Subject: Removed debug print statement. --- python/progress.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'python') diff --git a/python/progress.cc b/python/progress.cc index ab0deaef..027a51b4 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -208,7 +208,6 @@ void PyFetchProgress::Stop() * PyCbObj_END_ALLOW_THREADS to our previous * PyCbObj_BEGIN_ALLOW_THREADS (Python requires this!). */ - std::cout << "PyFetchProgress::STOP" << std::endl; PyCbObj_END_ALLOW_THREADS //std::cout << "Stop" << std::endl; pkgAcquireStatus::Stop(); -- cgit v1.2.3 From d28170024ef4d2f01fd9096a3ec785cf424c0846 Mon Sep 17 00:00:00 2001 From: Stephan Peijnik Date: Tue, 7 Jul 2009 18:44:10 +0200 Subject: Removed separate way of defining Py_ssize_t in progress.h. Now using method present in generic.h in progress.cc. --- debian/changelog | 2 +- python/progress.cc | 1 + python/progress.h | 6 ------ 3 files changed, 2 insertions(+), 7 deletions(-) (limited to 'python') diff --git a/debian/changelog b/debian/changelog index db6204ba..ce96cc9e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -python-apt (0.7.10.4) UNRELEASED; urgency=low +python-apt (0.7.11.0) UNRELEASED; urgency=low [ Stephan Peijnik ] * apt/progress/__init__.py: diff --git a/python/progress.cc b/python/progress.cc index cbb49671..94debe40 100644 --- a/python/progress.cc +++ b/python/progress.cc @@ -13,6 +13,7 @@ #include #include #include +#include "generic.h" #include "progress.h" // generic diff --git a/python/progress.h b/python/progress.h index 659eb113..29243bfc 100644 --- a/python/progress.h +++ b/python/progress.h @@ -15,12 +15,6 @@ #include #include -/* Python 2.4 compatibility */ -#if (PY_VERSION_HEX < 0x02050000) -typedef int Py_ssize_t; -#endif - - /* PyCbObj_BEGIN_ALLOW_THREADS and PyCbObj_END_ALLOW_THREADS are sligthly * modified versions of Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS. * Instead of storing the thread state in a function-local variable these -- cgit v1.2.3