* huge merge to bring the apt debian-sid and ubuntu branches as close

  as possible to apt--mvo again

5 files changed, 575 insertions, 3 deletions

diff --git a/methods/http.cc b/methods/http.cc
index 06b91686c..4b9622b00 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -55,11 +55,10 @@ using namespace std;
 string HttpMethod::FailFile;
 int HttpMethod::FailFd = -1;
 time_t HttpMethod::FailTime = 0;
-unsigned long PipelineDepth = 10;
+unsigned long PipelineDepth = 8;
 unsigned long TimeOut = 120;
 bool Debug = false;
 
-
 unsigned long CircleBuf::BwReadLimit=0;
 unsigned long CircleBuf::BwTickReadData=0;
 struct timeval CircleBuf::BwReadTick={0,0};
diff --git a/methods/https.cc b/methods/https.cc
new file mode 100644
index 000000000..b758e4ab3
--- /dev/null
+++ b/methods/https.cc
@@ -0,0 +1,249 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/
+// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
+/* ######################################################################
+
+   HTTPS Aquire Method - This is the HTTPS aquire method for APT.
+   
+   It uses libcurl
+
+   ##################################################################### */
+									/*}}}*/
+// Include Files							/*{{{*/
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/hashes.h>
+
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <utime.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <iostream>
+#include <apti18n.h>
+#include <sstream>
+
+#include "config.h"
+#include "https.h"
+
+									/*}}}*/
+using namespace std;
+
+size_t 
+HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
+{
+   HttpsMethod *me = (HttpsMethod *)userp;
+
+   if(me->File->Write(buffer, size*nmemb) != true)
+      return false;
+
+   return size*nmemb;
+}
+
+int 
+HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow, 
+			      double ultotal, double ulnow)
+{
+   HttpsMethod *me = (HttpsMethod *)clientp;
+   if(dltotal > 0 && me->Res.Size == 0) {
+      me->Res.Size = dltotal;
+      me->URIStart(me->Res);
+   }
+   return 0;
+}
+
+bool HttpsMethod::SetupProxy()
+{
+   URI ServerName = Queue->Uri;
+
+   // Determine the proxy setting
+   if (getenv("http_proxy") == 0)
+   {
+      string DefProxy = _config->Find("Acquire::http::Proxy");
+      string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
+      if (SpecificProxy.empty() == false)
+      {
+	 if (SpecificProxy == "DIRECT")
+	    Proxy = "";
+	 else
+	    Proxy = SpecificProxy;
+      }   
+      else
+	 Proxy = DefProxy;
+   }
+   
+   // Parse no_proxy, a , separated list of domains
+   if (getenv("no_proxy") != 0)
+   {
+      if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
+	 Proxy = "";
+   }
+   
+   // Determine what host and port to use based on the proxy settings
+   int Port = 0;
+   string Host;   
+   if (Proxy.empty() == true || Proxy.Host.empty() == true)
+   {
+   }
+   else
+   {
+      if (Proxy.Port != 0)
+	 curl_easy_setopt(curl, CURLOPT_PROXYPORT, Proxy.Port);
+      curl_easy_setopt(curl, CURLOPT_PROXY, Proxy.Host.c_str());
+   }
+}
+
+
+// HttpsMethod::Fetch - Fetch an item					/*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+   depth. */
+bool HttpsMethod::Fetch(FetchItem *Itm)
+{
+   stringstream ss;
+   struct stat SBuf;
+   struct curl_slist *headers=NULL;  
+   char curl_errorstr[CURL_ERROR_SIZE];
+
+   // TODO:
+   //       - http::Timeout
+   //       - http::Pipeline-Depth
+   //       - error checking/reporting
+   //       - more debug options? (CURLOPT_DEBUGFUNCTION?)
+
+   SetupProxy();
+
+   // callbacks
+   curl_easy_setopt(curl, CURLOPT_URL, Itm->Uri.c_str());
+   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
+   curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
+   curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
+   curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
+   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+   curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+
+   // FIXME: https: offer various options of verification
+   bool peer_verify = _config->FindB("Acquire::https::Verify-Peer", false);
+   curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, peer_verify);
+
+   // sslcert file
+   string pem = _config->Find("Acquire::https::SslCert","");
+   if(pem != "")
+      curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str());
+   
+   // CA-Dir
+   string certdir = _config->Find("Acquire::https::CaPath","");
+   if(certdir != "")
+      curl_easy_setopt(curl, CURLOPT_CAPATH, certdir.c_str());
+   
+   // Server-verify 
+   int verify = _config->FindI("Acquire::https::Verify-Host",2);
+   curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify);
+
+   // cache-control
+   if(_config->FindB("Acquire::http::No-Cache",false) == false)
+   {
+      // cache enabled
+      if (_config->FindB("Acquire::http::No-Store",false) == true)
+	 headers = curl_slist_append(headers,"Cache-Control: no-store");
+      ioprintf(ss, "Cache-Control: max-age=%u", _config->FindI("Acquire::http::Max-Age",0));
+      headers = curl_slist_append(headers, ss.str().c_str());
+   } else {
+      // cache disabled by user
+      headers = curl_slist_append(headers, "Cache-Control: no-cache");
+      headers = curl_slist_append(headers, "Pragma: no-cache");
+   }
+   curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
+   // set time values
+   curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
+   curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified);
+
+   // speed limit
+   int dlLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
+   if (dlLimit > 0)
+      curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, dlLimit);
+
+   // set header
+   curl_easy_setopt(curl, CURLOPT_USERAGENT,"Debian APT-CURL/1.0 ("VERSION")");
+
+   // debug
+   if(_config->FindB("Debug::Acquire::https", false))
+      curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
+
+   // error handling
+   curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
+
+   // In this case we send an if-range query with a range header
+  if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
+     curl_easy_setopt(curl, CURLOPT_RESUME_FROM, (long)SBuf.st_size);
+
+   // go for it - if the file exists, append on it
+   File = new FileFd(Itm->DestFile, FileFd::WriteAny);
+   File->Seek(File->Size());
+   
+   // keep apt updated
+   Res.Filename = Itm->DestFile;
+
+   // get it!
+   CURLcode success = curl_easy_perform(curl);
+
+
+   // cleanup
+   if(success != 0) {
+      _error->Error(curl_errorstr);
+      Fail();
+      return true;
+   }
+
+   if (Res.Size == 0)
+      Res.Size = File->Size();
+
+   // check the downloaded result
+   struct stat Buf;
+   if (stat(File->Name().c_str(),&Buf) == 0)
+   {
+      Res.Size = Buf.st_size;
+      Res.Filename = File->Name();
+      Res.LastModified = Buf.st_mtime;
+      Res.IMSHit = false;
+      if (Itm->LastModified != 0 && Buf.st_mtime >= Itm->LastModified)
+      {
+	 Res.IMSHit = true;
+	 Res.LastModified = Itm->LastModified;
+      }
+   }
+
+   // take hashes
+   Hashes Hash;
+   FileFd Fd(Res.Filename, FileFd::ReadOnly);
+   Hash.AddFD(Fd.Fd(), Fd.Size());
+   Res.TakeHashes(Hash);
+   
+   // keep apt updated
+   URIDone(Res);
+
+   // cleanup
+   File->Close();
+   Res.Size = 0;
+   delete File;
+   curl_slist_free_all(headers);
+
+   return true;
+};
+
+int main()
+{
+   setlocale(LC_ALL, "");
+
+   HttpsMethod Mth;
+   curl_global_init(CURL_GLOBAL_SSL) ;
+
+   return Mth.Run();
+}
+
+
diff --git a/methods/https.h b/methods/https.h
new file mode 100644
index 000000000..6620a10fc
--- /dev/null
+++ b/methods/https.h
@@ -0,0 +1,48 @@
+// -*- mode: cpp; mode: fold -*-
+// Description								/*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
+// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $
+/* ######################################################################
+
+   HTTP Aquire Method - This is the HTTP aquire method for APT.
+
+   ##################################################################### */
+									/*}}}*/
+
+#ifndef APT_HTTP_H
+#define APT_HTTP_H
+
+#define MAXLEN 360
+
+#include <iostream>
+#include <curl/curl.h>
+
+using std::cout;
+using std::endl;
+
+class HttpsMethod;
+
+
+class HttpsMethod : public pkgAcqMethod
+{
+
+   virtual bool Fetch(FetchItem *);
+   static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
+   static int progress_callback(void *clientp, double dltotal, double dlnow, 
+				double ultotal, double ulnow);
+   bool SetupProxy();
+   CURL *curl;
+   FetchResult Res;
+
+   public:
+   FileFd *File;
+      
+   HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig) 
+   {
+      File = 0;
+      curl = curl_easy_init();
+   };
+};
+
+URI Proxy;
+
+#endif
diff --git a/methods/makefile b/methods/makefile
index 3f561a2c3..f178cbbea 100644
--- a/methods/makefile
+++ b/methods/makefile
@@ -7,7 +7,7 @@ include ../buildlib/defaults.mak
 BIN := $(BIN)/methods
 
 # FIXME..
-LIB_APT_PKG_MAJOR = 3.12
+LIB_APT_PKG_MAJOR = 4.4
 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR)
 
 # The file method
@@ -52,6 +52,13 @@ LIB_MAKES = apt-pkg/makefile
 SOURCE = http.cc rfc2553emu.cc connect.cc
 include $(PROGRAM_H)
 
+# The https method
+PROGRAM=https
+SLIBS = -lapt-pkg -lcurl
+LIB_MAKES = apt-pkg/makefile
+SOURCE = https.cc
+include $(PROGRAM_H)
+
 # The ftp method
 PROGRAM=ftp
 SLIBS = -lapt-pkg $(SOCKETLIBS)
@@ -59,6 +66,13 @@ LIB_MAKES = apt-pkg/makefile
 SOURCE = ftp.cc rfc2553emu.cc connect.cc
 include $(PROGRAM_H)
 
+# The rred method
+PROGRAM=rred
+SLIBS = -lapt-pkg $(SOCKETLIBS)
+LIB_MAKES = apt-pkg/makefile
+SOURCE = rred.cc
+include $(PROGRAM_H)
+
 # The rsh method
 PROGRAM=rsh
 SLIBS = -lapt-pkg
diff --git a/methods/rred.cc b/methods/rred.cc
new file mode 100644
index 000000000..6fa57f3a6
--- /dev/null
+++ b/methods/rred.cc
@@ -0,0 +1,262 @@
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/acquire-method.h>
+#include <apt-pkg/strutl.h>
+#include <apt-pkg/hashes.h>
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <utime.h>
+#include <stdio.h>
+#include <errno.h>
+#include <apti18n.h>
+
+/* this method implements a patch functionality similar to "patch --ed" that is
+ * used by the "tiffany" incremental packages download stuff. it differs from 
+ * "ed" insofar that it is way more restricted (and therefore secure). in the
+ * moment only the "c", "a" and "d" commands of ed are implemented (diff 
+ * doesn't output any other). additionally the records must be reverse sorted 
+ * by line number and may not overlap (diff *seems* to produce this kind of 
+ * output). 
+ * */
+
+const char *Prog;
+
+class RredMethod : public pkgAcqMethod
+{
+   bool Debug;
+   // the size of this doesn't really matter (except for performance)    
+   const static int BUF_SIZE = 1024;
+   // the ed commands
+   enum Mode {MODE_CHANGED, MODE_DELETED, MODE_ADDED};
+   // return values
+   enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE};
+   // this applies a single hunk, it uses a tail recursion to 
+   // reverse the hunks in the file
+   int ed_rec(FILE *ed_cmds, FILE *in_file, FILE *out_file, int line, 
+      char *buffer, unsigned int bufsize, Hashes *hash);
+   // apply a patch file
+   int ed_file(FILE *ed_cmds, FILE *in_file, FILE *out_file, Hashes *hash);
+   // the methods main method
+   virtual bool Fetch(FetchItem *Itm);
+   
+   public:
+   
+   RredMethod() : pkgAcqMethod("1.1",SingleInstance | SendConfig) {};
+};
+
+int RredMethod::ed_rec(FILE *ed_cmds, FILE *in_file, FILE *out_file, int line, 
+      char *buffer, unsigned int bufsize, Hashes *hash) {
+   int pos;
+   int startline;
+   int stopline;
+   int mode;
+   int written;
+   char *idx;
+
+   /* get the current command and parse it*/
+   if (fgets(buffer, bufsize, ed_cmds) == NULL) {
+      return line;
+   }
+   startline = strtol(buffer, &idx, 10);
+   if (startline < line) {
+      return ED_ORDERING;
+   }
+   if (*idx == ',') {
+      idx++;
+      stopline = strtol(idx, &idx, 10);
+   }
+   else {
+      stopline = startline;
+   }
+   if (*idx == 'c') {
+      mode = MODE_CHANGED;
+	   if (Debug == true) {
+		   std::clog << "changing from line " << startline 
+			     << " to " << stopline << std::endl;
+	   }
+   }
+   else if (*idx == 'a') {
+      mode = MODE_ADDED;
+	   if (Debug == true) {
+		   std::clog << "adding after line " << startline << std::endl;
+	   }
+   }
+   else if (*idx == 'd') {
+      mode = MODE_DELETED;
+	   if (Debug == true) {
+		   std::clog << "deleting from line " << startline 
+			     <<  " to " << stopline << std::endl;
+	   }
+   }
+   else {
+      return ED_PARSER;
+   }
+   /* get the current position */
+   pos = ftell(ed_cmds);
+   /* if this is add or change then go to the next full stop */
+   if ((mode == MODE_CHANGED) || (mode == MODE_ADDED)) {
+      do {
+         fgets(buffer, bufsize, ed_cmds);
+         while ((strlen(buffer) == (bufsize - 1)) 
+               && (buffer[bufsize - 2] != '\n')) {
+            fgets(buffer, bufsize, ed_cmds);
+            buffer[0] = ' ';
+         }
+      } while (strncmp(buffer, ".", 1) != 0);
+   }
+   /* do the recursive call */
+   line = ed_rec(ed_cmds, in_file, out_file, line, buffer, bufsize, 
+         hash);
+   /* pass on errors */
+   if (line < 0) {
+      return line;
+   }
+   /* apply our hunk */
+   fseek(ed_cmds, pos, SEEK_SET); 
+   /* first wind to the current position */
+   if (mode != MODE_ADDED) {
+      startline -= 1;
+   }
+   while (line < startline) {
+      fgets(buffer, bufsize, in_file);
+      written = fwrite(buffer, 1, strlen(buffer), out_file);
+      hash->Add((unsigned char*)buffer, written);
+      while ((strlen(buffer) == (bufsize - 1)) 
+            && (buffer[bufsize - 2] != '\n')) {
+         fgets(buffer, bufsize, in_file);
+         written = fwrite(buffer, 1, strlen(buffer), out_file);
+         hash->Add((unsigned char*)buffer, written);
+      }
+      line++;
+   }
+   /* include from ed script */
+   if ((mode == MODE_ADDED) || (mode == MODE_CHANGED)) {
+      do {
+         fgets(buffer, bufsize, ed_cmds);
+         if (strncmp(buffer, ".", 1) != 0) {
+            written = fwrite(buffer, 1, strlen(buffer), out_file);
+            hash->Add((unsigned char*)buffer, written);
+            while ((strlen(buffer) == (bufsize - 1)) 
+                  && (buffer[bufsize - 2] != '\n')) {
+               fgets(buffer, bufsize, ed_cmds);
+               written = fwrite(buffer, 1, strlen(buffer), out_file);
+               hash->Add((unsigned char*)buffer, written);
+            }
+         }
+         else {
+            break;
+         }
+      } while (1);
+   }
+   /* ignore the corresponding number of lines from input */
+   if ((mode == MODE_DELETED) || (mode == MODE_CHANGED)) {
+      while (line < stopline) {
+         fgets(buffer, bufsize, in_file);
+         while ((strlen(buffer) == (bufsize - 1)) 
+               && (buffer[bufsize - 2] != '\n')) {
+            fgets(buffer, bufsize, in_file);
+         }
+         line++;
+      }
+   }
+   return line;
+}
+
+int RredMethod::ed_file(FILE *ed_cmds, FILE *in_file, FILE *out_file, 
+      Hashes *hash) {
+   char buffer[BUF_SIZE];
+   int result;
+   int written;
+   
+   /* we do a tail recursion to read the commands in the right order */
+   result = ed_rec(ed_cmds, in_file, out_file, 0, buffer, BUF_SIZE, 
+         hash);
+   
+   /* read the rest from infile */
+   if (result > 0) {
+      while (fgets(buffer, BUF_SIZE, in_file) != NULL) {
+         written = fwrite(buffer, 1, strlen(buffer), out_file);
+         hash->Add((unsigned char*)buffer, written);
+      }
+   }
+   else {
+      return ED_FAILURE;
+   }
+   return ED_OK;
+}
+
+
+bool RredMethod::Fetch(FetchItem *Itm)
+{
+   Debug = _config->FindB("Debug::pkgAcquire::RRed",false);
+   URI Get = Itm->Uri;
+   string Path = Get.Host + Get.Path; // To account for relative paths
+   // Path contains the filename to patch
+   FetchResult Res;
+   Res.Filename = Itm->DestFile;
+   URIStart(Res);
+   // Res.Filename the destination filename
+
+   if (Debug == true) 
+      std::clog << "Patching " << Path << " with " << Path 
+         << ".ed and putting result into " << Itm->DestFile << std::endl;
+   // Open the source and destination files (the d'tor of FileFd will do 
+   // the cleanup/closing of the fds)
+   FileFd From(Path,FileFd::ReadOnly);
+   FileFd Patch(Path+".ed",FileFd::ReadOnly);
+   FileFd To(Itm->DestFile,FileFd::WriteEmpty);   
+   To.EraseOnFailure();
+   if (_error->PendingError() == true)
+      return false;
+   
+   Hashes Hash;
+   FILE* fFrom = fdopen(From.Fd(), "r");
+   FILE* fPatch = fdopen(Patch.Fd(), "r");
+   FILE* fTo = fdopen(To.Fd(), "w");
+   // now do the actual patching
+   if (ed_file(fPatch, fFrom, fTo, &Hash) != ED_OK) {
+     _error->Errno("rred", _("Could not patch file"));  
+      return false;
+   }
+
+   // write out the result
+   fflush(fFrom);
+   fflush(fPatch);
+   fflush(fTo);
+   From.Close();
+   Patch.Close();
+   To.Close();
+
+   // Transfer the modification times
+   struct stat Buf;
+   if (stat(Path.c_str(),&Buf) != 0)
+      return _error->Errno("stat",_("Failed to stat"));
+
+   struct utimbuf TimeBuf;
+   TimeBuf.actime = Buf.st_atime;
+   TimeBuf.modtime = Buf.st_mtime;
+   if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0)
+      return _error->Errno("utime",_("Failed to set modification time"));
+
+   if (stat(Itm->DestFile.c_str(),&Buf) != 0)
+      return _error->Errno("stat",_("Failed to stat"));
+
+   // return done
+   Res.LastModified = Buf.st_mtime;
+   Res.Size = Buf.st_size;
+   Res.TakeHashes(Hash);
+   URIDone(Res);
+
+   return true;
+}
+
+int main(int argc, char *argv[])
+{
+   RredMethod Mth;
+
+   Prog = strrchr(argv[0],'/');
+   Prog++;
+   
+   return Mth.Run();
+}