From 2f6557b96c08c1adebf3b1e292ae636a27f624d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 30 Nov 2010 18:57:42 +0100 Subject: add the possibility to disable only the progress reporting stuff as the quiet level 1 does this, but also disables other stuff we might want to test against in a testcase --- doc/examples/configure-index | 1 + 1 file changed, 1 insertion(+) (limited to 'doc') diff --git a/doc/examples/configure-index b/doc/examples/configure-index index c4c2acb64..6c078d75f 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -17,6 +17,7 @@ */ quiet "0"; +quiet::NoUpdate "true"; // never update progress information - included in -q=1 // Options for APT in general APT -- cgit v1.2.3 From 1fc0d435891dc9496a71a680032918dab2372e6d Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Thu, 13 Jan 2011 00:01:27 +0100 Subject: * doc/apt.conf.5.xml: - fix multipl{y,e} spelling error reported by Jakub Wilk (Closes: #607636) --- debian/changelog | 4 +++- doc/apt.conf.5.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'doc') diff --git a/debian/changelog b/debian/changelog index dde28c635..6028cc8f4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -27,8 +27,10 @@ apt (0.8.11+wheezy) unstable; urgency=low * deb/dpkgpm.cc: - fix popen/fclose mismatch reported by cppcheck. Thanks to Petter Reinholdtsen for report and patch! (Closes: #607803) + * doc/apt.conf.5.xml: + - fix multipl{y,e} spelling error reported by Jakub Wilk (Closes: #607636) - -- David Kalnischkies Wed, 12 Jan 2011 23:53:32 +0100 + -- David Kalnischkies Wed, 12 Jan 2011 23:59:38 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index f00baacea..a19d85dbc 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -618,7 +618,7 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; dpkg trigger usage (and related options) APT can call dpkg in a way so it can make aggressive use of triggers over - multiply calls of dpkg. Without further options dpkg will use triggers only in between his + multiple calls of dpkg. Without further options dpkg will use triggers only in between his own run. Activating these options can therefore decrease the time needed to perform the install / upgrade. Note that it is intended to activate these options per default in the future, but as it changes the way APT calling dpkg drastically it needs a lot more testing. -- cgit v1.2.3 From 39ad9b28cf43f121b410bd5874eebe9934c81482 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 13 Jan 2011 23:47:01 +0100 Subject: * doc/apt-get.8.xml: - remove duplicated mentioning of --install-recommends --- debian/changelog | 4 +++- doc/apt-get.8.xml | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'doc') diff --git a/debian/changelog b/debian/changelog index ca5d4f235..4e50172c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -35,8 +35,10 @@ apt (0.8.11+wheezy) unstable; urgency=low Thanks to Nobuhiro Hayashi for the patch! (Closes: #330162) * apt-pkg/cacheiterator.h: - do not segfault if cache is not build (Closes: #254770) + * doc/apt-get.8.xml: + - remove duplicated mentioning of --install-recommends - -- David Kalnischkies Thu, 13 Jan 2011 23:19:03 +0100 + -- David Kalnischkies Thu, 13 Jan 2011 23:46:23 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml index 57306c03f..1f14c6bd5 100644 --- a/doc/apt-get.8.xml +++ b/doc/apt-get.8.xml @@ -411,14 +411,6 @@ Configuration Item: APT::Get::Compile. - - Also install recommended packages. - - - - Do not install recommended packages. - - Ignore package Holds; This causes apt-get to ignore a hold placed on a package. This may be useful in conjunction with -- cgit v1.2.3 From 8b39570568ce96d3afd84c397b6722bc12104e9e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 14 Jan 2011 01:50:55 +0100 Subject: remove duplicated text and refer to the sources.list manpage instead --- doc/files.sgml | 80 +++------------------------------------------------------- 1 file changed, 4 insertions(+), 76 deletions(-) (limited to 'doc') diff --git a/doc/files.sgml b/doc/files.sgml index 108e73670..2d0ae4a44 100644 --- a/doc/files.sgml +++ b/doc/files.sgml @@ -118,82 +118,10 @@ fastest source listed first. The format of each line is:

The first item, type, indicates the format for the remainder of the line. It is designed to indicate the structure of the distribution -the line is talking about. Currently the only defined value is deb -which indicates a standard debian archive with a dists dir. - -The deb Type -

- The deb type is to be a typical two level debian distributions, - dist/distribution/component. Typically distribution - is one of stable, unstable or testing while component is one of main, - contrib, non-free or non-us. The format for the deb line is as follows: - -

- deb uri distribution component - [component ...] - -

- uri for the deb type must specify the base of the - debian distribution. APT will automatically generate the proper longer - URIs to get the information it needs. distribution can specify - an exact path, in this case the components must be omitted and - distribution must end in a slash. - -

- Since only one distribution can be specified per deb line it may be - necessary to list a number of deb lines for the same URI. APT will - sort the URI list after it has generated a complete set to allow - connection reuse. It is important to order things in the sourcelist - from most preferred to least preferred (fastest to slowest). - - -URI specification -

-URIs in the source list support a large number of access schemes which -are listed in the sources.list manpage and can be further extended by -transport binaries placed in /usr/lib/apt/methods. The most important -builtin schemes are: - - -cdrom - The cdrom scheme is special in that If Modified Since queries are never - performed and that APT knows how to match a cdrom to the name it - was given when first inserted. APT also knows all of the possible - mount points the cdrom drives and that the user should be prompted - to insert a CD if it cannot be found. The path is relative to an - arbitrary mount point (of APT's choosing) and must not start with a - slash. The first pathname component is the given name and is purely - descriptive and of the users choice. However, if a file in the root of - the cdrom is called '.disk/info' its contents will be used instead of - prompting. The name serves as a tag for the cdrom and should be unique. - - cdrom:Debian 1.3/debian - - -http - This scheme specifies a HTTP server for the debian archive. HTTP is preferred - over FTP because If Modified Since queries against the Package file are - possible as well as deep pipelining and resume capabilities. - - http://www.debian.org/archive - - -ftp - This scheme specifies a FTP connection to the server. FTP is limited because - there is no support for IMS and is hard to proxy over firewalls. - - ftp://ftp.debian.org/debian - - -file - The file scheme allows an arbitrary directory in the file system to be - considered as a debian archive. This is useful for NFS mounts and - local mirrors/archives. - - file:/var/debian - - - +the line is talking about. Currently the only defined values are deb +and deb-src which indicate a standard debian (source) archive with a +dists directory. More about these types and the URI specification can be found +in the sources.list manpage. Hashing the URI

-- cgit v1.2.3 From 3805b0a7ff4fa4b18367f0328a744402d0d21d3b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 14 Jan 2011 01:54:24 +0100 Subject: * doc/sources.list.5.xml: - remove obsolete references to non-us (Closes: #594495) --- debian/changelog | 4 +++- doc/sources.list.5.xml | 30 +++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) (limited to 'doc') diff --git a/debian/changelog b/debian/changelog index 4e50172c6..326ac7c98 100644 --- a/debian/changelog +++ b/debian/changelog @@ -37,8 +37,10 @@ apt (0.8.11+wheezy) unstable; urgency=low - do not segfault if cache is not build (Closes: #254770) * doc/apt-get.8.xml: - remove duplicated mentioning of --install-recommends + * doc/sources.list.5.xml: + - remove obsolete references to non-us (Closes: #594495) - -- David Kalnischkies Thu, 13 Jan 2011 23:46:23 +0100 + -- David Kalnischkies Fri, 14 Jan 2011 01:51:50 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index 6029a7457..212ed6d98 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -63,11 +63,11 @@ The deb and deb-src types The deb type describes a typical two-level Debian archive, distribution/component. Typically, - distribution is generally one of - stable unstable or - testing while component is one of main - contrib non-free or - non-us. The + distribution is generally an archivename like + stable or testing or a codename like + &stable-codename; or &testing-codename; + while component is one of main contrib or + non-free. The deb-src type describes a debian distribution's source code in the same form as the deb type. A deb-src line is required to fetch source indexes. @@ -218,18 +218,14 @@ deb http://security.debian.org/ &stable-codename;/updates main contrib non-free a single FTP session will be used for both resource lines. deb ftp://ftp.debian.org/debian unstable contrib - Uses HTTP to access the archive at nonus.debian.org, under the - debian-non-US directory. - deb http://nonus.debian.org/debian-non-US stable/non-US main contrib non-free - - Uses HTTP to access the archive at nonus.debian.org, under the - debian-non-US directory, and uses only files found under - unstable/binary-i386 on i386 machines, - unstable/binary-m68k on m68k, and so - forth for other supported architectures. [Note this example only - illustrates how to use the substitution variable; non-us is no longer - structured like this] - deb http://ftp.de.debian.org/debian-non-US unstable/binary-$(ARCH)/ + Uses HTTP to access the archive at ftp.tlh.debian.org, under the + universe directory, and uses only files found under + unstable/binary-i386 on i386 machines, + unstable/binary-amd64 on amd64, and so + forth for other supported architectures. [Note this example only + illustrates how to use the substitution variable; official debian + archives are not structured like this] + deb http://ftp.tlh.debian.org/universe unstable/binary-$(ARCH)/ -- cgit v1.2.3 From fe0f7911b650918e1d511b3453664a07f6d966d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 20 Jan 2011 14:53:52 +0100 Subject: - try downloading clearsigned InRelease before trying Release.gpg * apt-pkg/deb/deblistparser.cc: - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles --- apt-pkg/acquire-item.cc | 54 ++++++- apt-pkg/acquire-item.h | 33 +++++ apt-pkg/deb/debindexfile.cc | 8 +- apt-pkg/deb/deblistparser.cc | 114 ++++++++++----- apt-pkg/deb/debmetaindex.cc | 34 +++-- apt-pkg/indexcopy.cc | 3 +- apt-pkg/indexrecords.cc | 17 ++- apt-pkg/tagfile.cc | 10 +- apt-pkg/tagfile.h | 2 + debian/changelog | 5 +- doc/apt-secure.8.xml | 5 +- test/integration/Packages-releasefile-verification | 18 +++ .../Packages-releasefile-verification-new | 21 +++ test/integration/framework | 7 + test/integration/marvinparanoid.pub | Bin 0 -> 629 bytes test/integration/marvinparanoid.sec | Bin 0 -> 1280 bytes .../test-bug-595691-empty-and-broken-archive-files | 32 ++--- test/integration/test-releasefile-verification | 160 +++++++++++++++++++++ 18 files changed, 430 insertions(+), 93 deletions(-) create mode 100644 test/integration/Packages-releasefile-verification create mode 100644 test/integration/Packages-releasefile-verification-new create mode 100644 test/integration/marvinparanoid.pub create mode 100644 test/integration/marvinparanoid.sec create mode 100755 test/integration/test-releasefile-verification (limited to 'doc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d4e90b552..2cd6ab359 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1077,6 +1077,8 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, /*{{{* { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + SigFile = FinalFile; Rename(DestFile,FinalFile); chmod(FinalFile.c_str(),0644); DestFile = FinalFile; @@ -1110,6 +1112,8 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + SigFile = FinalFile; DestFile = FinalFile; } Complete = true; @@ -1141,6 +1145,10 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ // Download further indexes with verification QueueIndexes(true); + // is it a clearsigned MetaIndex file? + if (DestFile == SigFile) + return; + // Done, move signature file into position string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI) + ".gpg"; @@ -1300,13 +1308,20 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) if (AuthPass == true) { // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists") + - "partial/" + URItoFileName(RealURI) + ".gpg.reverify"; + string LastGoodSigFile = _config->FindDir("Dir::State::lists"); + if (DestFile == SigFile) + LastGoodSigFile.append(URItoFileName(RealURI)); + else + LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify"); + if(FileExists(LastGoodSigFile)) { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(RealURI) + ".gpg"; - Rename(LastGoodSigFile,VerifiedSigFile); + if (DestFile != SigFile) + { + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(RealURI) + ".gpg"; + Rename(LastGoodSigFile,VerifiedSigFile); + } Status = StatTransientNetworkError; _error->Warning(_("A error occurred during the signature " "verification. The repository is not updated " @@ -1330,6 +1345,35 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) QueueIndexes(false); } /*}}}*/ +pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ + string const &URI, string const &URIDesc, string const &ShortDesc, + string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, + string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, + const vector* IndexTargets, + indexRecords* MetaIndexParser) : + pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) +{ + SigFile = DestFile; +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + if (AuthPass == false) + { + new pkgAcqMetaSig(Owner, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + IndexTargets, MetaIndexParser); + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) + Dequeue(); + } + else + pkgAcqMetaIndex::Failed(Message, Cnf); +} + /*}}}*/ // AcqArchive::AcqArchive - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This just sets up the initial fetch environment and queues the first diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index d97a96a0f..581761e32 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -772,6 +772,39 @@ class pkgAcqMetaIndex : public pkgAcquire::Item indexRecords* MetaIndexParser); }; /*}}}*/ +/** \brief An item repsonsible for downloading clearsigned metaindexes {{{*/ +class pkgAcqMetaClearSig : public pkgAcqMetaIndex +{ + /** \brief The URI of the meta-index file for the detached signature */ + string MetaIndexURI; + + /** \brief A "URI-style" description of the meta-index file */ + string MetaIndexURIDesc; + + /** \brief A brief description of the meta-index file */ + string MetaIndexShortDesc; + + /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */ + string MetaSigURI; + + /** \brief A "URI-style" description of the meta-signature file */ + string MetaSigURIDesc; + + /** \brief A brief description of the meta-signature file */ + string MetaSigShortDesc; + +public: + void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Create a new pkgAcqMetaClearSig. */ + pkgAcqMetaClearSig(pkgAcquire *Owner, + string const &URI, string const &URIDesc, string const &ShortDesc, + string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, + string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, + const vector* IndexTargets, + indexRecords* MetaIndexParser); +}; + /*}}}*/ /** \brief An item that is responsible for fetching a package file. {{{ * * If the package file already exists in the cache, nothing will be diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index af1209ccb..9961b5ae4 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -324,8 +324,14 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const return _error->Error("Problem with MergeList %s",PackageFile.c_str()); // Check the release file - string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release"); + string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease"); + bool releaseExists = false; if (FileExists(ReleaseFile) == true) + releaseExists = true; + else + ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release"); + + if (releaseExists == true || FileExists(ReleaseFile) == true) { FileFd Rel(ReleaseFile,FileFd::ReadOnly); if (_error->PendingError() == true) diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 1b3bfd6ae..9201e6a54 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -783,45 +783,89 @@ bool debListParser::Step() bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, FileFd &File, string component) { - pkgTagFile Tags(&File, File.Size() + 256); // XXX - pkgTagSection Section; - if (Tags.Step(Section) == false) - return false; - - // FIXME: Do we need it now for multi-arch? - // mvo: I don't think we need to fill that in (it's unused since apt-0.6) -// FileI->Architecture = WriteUniqString(Arch); - // apt-secure does no longer download individual (per-section) Release // file. to provide Component pinning we use the section name now FileI->Component = WriteUniqString(component); - const char *Start; - const char *Stop; - if (Section.Find("Suite",Start,Stop) == true) - FileI->Archive = WriteUniqString(Start,Stop - Start); - if (Section.Find("Component",Start,Stop) == true) - FileI->Component = WriteUniqString(Start,Stop - Start); - if (Section.Find("Version",Start,Stop) == true) - FileI->Version = WriteUniqString(Start,Stop - Start); - if (Section.Find("Origin",Start,Stop) == true) - FileI->Origin = WriteUniqString(Start,Stop - Start); - if (Section.Find("Codename",Start,Stop) == true) - FileI->Codename = WriteUniqString(Start,Stop - Start); - if (Section.Find("Label",Start,Stop) == true) - FileI->Label = WriteUniqString(Start,Stop - Start); - if (Section.Find("Architecture",Start,Stop) == true) - FileI->Architecture = WriteUniqString(Start,Stop - Start); - - if (Section.FindFlag("NotAutomatic",FileI->Flags, - pkgCache::Flag::NotAutomatic) == false) - _error->Warning("Bad NotAutomatic flag"); - if (Section.FindFlag("ButAutomaticUpgrades",FileI->Flags, - pkgCache::Flag::ButAutomaticUpgrades) == false) - _error->Warning("Bad ButAutomaticUpgrades flag"); - // overrule the NotAutomatic setting if needed as they are both present for compatibility - else if ((FileI->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades) - FileI->Flags &= ~pkgCache::Flag::NotAutomatic; + FILE* release = fdopen(dup(File.Fd()), "r"); + if (release == NULL) + return false; + + char buffer[101]; + bool gpgClose = false; + while (fgets(buffer, sizeof(buffer), release) != NULL) + { + size_t len = 0; + + // Skip empty lines + for (; buffer[len] == '\r' && buffer[len] == '\n'; ++len); + if (buffer[len] == '\0') + continue; + + // only evalute the first GPG section + if (strncmp("-----", buffer, 5) == 0) + { + if (gpgClose == true) + break; + gpgClose = true; + continue; + } + + // seperate the tag from the data + for (; buffer[len] != ':' && buffer[len] != '\0'; ++len); + if (buffer[len] == '\0') + continue; + char* dataStart = buffer + len; + for (++dataStart; *dataStart == ' '; ++dataStart); + char* dataEnd = dataStart; + for (++dataEnd; *dataEnd != '\0'; ++dataEnd); + + // which datastorage need to be updated + map_ptrloc* writeTo = NULL; + if (buffer[0] == ' ') + ; + #define APT_PARSER_WRITETO(X, Y) else if (strncmp(Y, buffer, len) == 0) writeTo = &X; + APT_PARSER_WRITETO(FileI->Archive, "Suite") + APT_PARSER_WRITETO(FileI->Component, "Component") + APT_PARSER_WRITETO(FileI->Version, "Version") + APT_PARSER_WRITETO(FileI->Origin, "Origin") + APT_PARSER_WRITETO(FileI->Codename, "Codename") + APT_PARSER_WRITETO(FileI->Label, "Label") + #undef APT_PARSER_WRITETO + #define APT_PARSER_FLAGIT(X) else if (strncmp(#X, buffer, len) == 0) \ + pkgTagSection::FindFlag(FileI->Flags, pkgCache::Flag:: X, dataStart, dataEnd-1); + APT_PARSER_FLAGIT(NotAutomatic) + APT_PARSER_FLAGIT(ButAutomaticUpgrades) + #undef APT_PARSER_FLAGIT + + // load all data from the line and save it + string data; + if (writeTo != NULL) + data.append(dataStart, dataEnd); + if (sizeof(buffer) - 1 == (dataEnd - buffer)) + { + while (fgets(buffer, sizeof(buffer), release) != NULL) + { + if (writeTo != NULL) + data.append(buffer); + if (strlen(buffer) != sizeof(buffer) - 1) + break; + } + } + if (writeTo != NULL) + { + // remove spaces and stuff from the end of the data line + for (std::string::reverse_iterator s = data.rbegin(); + s != data.rend(); ++s) + { + if (*s != '\r' && *s != '\n' && *s != ' ') + break; + *s = '\0'; + } + *writeTo = WriteUniqString(data); + } + } + fclose(release); return !_error->PendingError(); } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 717d0bcde..e2c680b14 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -182,21 +182,15 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, HashString()); } - // this is normally created in pkgAcqMetaSig, but if we run - // in --print-uris mode, we add it here - new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"), - MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), - ComputeIndexTargets(), - new indexRecords (Dist)); - } - new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"), - MetaIndexInfo("Release.gpg"), "Release.gpg", - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - ComputeIndexTargets(), - new indexRecords (Dist)); + new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"), + MetaIndexInfo("InRelease"), "InRelease", + MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", + ComputeIndexTargets(), + new indexRecords (Dist)); + // Queue the translations std::vector const lang = APT::Configuration::getLanguages(true); @@ -224,16 +218,20 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const bool debReleaseIndex::IsTrusted() const { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("Release")) + ".gpg"; - if(_config->FindB("APT::Authentication::TrustCDROM", false)) if(URI.substr(0,strlen("cdrom:")) == "cdrom:") return true; - + + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(MetaIndexURI("Release")) + ".gpg"; + if (FileExists(VerifiedSigFile)) return true; - return false; + + VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(MetaIndexURI("InRelease")); + + return FileExists(VerifiedSigFile); } vector *debReleaseIndex::GetIndexFiles() { diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index f88d51fc5..c2ee1c347 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -722,7 +722,8 @@ bool SigVerify::RunGPGV(std::string const &File, std::string const &FileGPG, } Args.push_back(FileGPG.c_str()); - Args.push_back(File.c_str()); + if (FileGPG != File) + Args.push_back(File.c_str()); Args.push_back(NULL); if (Debug == true) diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index eb9a36866..10e154ad2 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -55,14 +55,17 @@ bool indexRecords::Load(const string Filename) /*{{{*/ } pkgTagSection Section; - if (TagFile.Step(Section) == false) - { - strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); - return false; - } - const char *Start, *End; - Section.Get (Start, End, 0); + // Skip over sections beginning with ----- as this is an idicator for clearsigns + do { + if (TagFile.Step(Section) == false) + { + strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); + return false; + } + + Section.Get (Start, End, 0); + } while (End - Start > 5 && strncmp(Start, "-----", 5) == 0); Suite = Section.FindS("Suite"); Dist = Section.FindS("Codename"); diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 96a681bec..4a2f3f7e6 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -399,9 +399,13 @@ bool pkgTagSection::FindFlag(const char *Tag,unsigned long &Flags, const char *Stop; if (Find(Tag,Start,Stop) == false) return true; - - switch (StringToBool(string(Start,Stop))) - { + return FindFlag(Flags, Flag, Start, Stop); +} +bool const pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag, + char const* Start, char const* Stop) +{ + switch (StringToBool(string(Start, Stop))) + { case 0: Flags &= ~Flag; return true; diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 6891c1d81..61491aa04 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -60,6 +60,8 @@ class pkgTagSection unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const; bool FindFlag(const char *Tag,unsigned long &Flags, unsigned long Flag) const; + bool static const FindFlag(unsigned long &Flags, unsigned long Flag, + const char* Start, const char* Stop); bool Scan(const char *Start,unsigned long MaxLength); inline unsigned long size() const {return Stop - Section;}; void Trim(); diff --git a/debian/changelog b/debian/changelog index b74508804..59f516751 100644 --- a/debian/changelog +++ b/debian/changelog @@ -53,11 +53,14 @@ apt (0.8.11+wheezy) unstable; urgency=low - operate optional on gzip compressed pdiffs * apt-pkg/acquire-item.cc: - don't uncompress downloaded pdiff files before feeding it to rred + - try downloading clearsigned InRelease before trying Release.gpg * cmdline/apt-key: - don't set trustdb-name as non-root so 'list' and 'finger' can be used without being root (Closes: #393005, #592107) + * apt-pkg/deb/deblistparser.cc: + - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles - -- David Kalnischkies Sun, 16 Jan 2011 17:23:28 +0100 + -- David Kalnischkies Thu, 20 Jan 2011 14:52:32 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/apt-secure.8.xml b/doc/apt-secure.8.xml index f345c3f89..f8ff678b9 100644 --- a/doc/apt-secure.8.xml +++ b/doc/apt-secure.8.xml @@ -148,8 +148,8 @@ (you should make sure you are using a trusted communication channel when retrieving it), add it with apt-key and then run apt-get update so that apt can download - and verify the Release.gpg files from the archives you - have configured. + and verify the InRelease or Release.gpg + files from the archives you have configured. @@ -166,6 +166,7 @@ (provided in apt-utils). Sign it. You can do this by running + gpg --clearsign -o InRelease Release and gpg -abs -o Release.gpg Release. Publish the key fingerprint, diff --git a/test/integration/Packages-releasefile-verification b/test/integration/Packages-releasefile-verification new file mode 100644 index 000000000..29a385f4f --- /dev/null +++ b/test/integration/Packages-releasefile-verification @@ -0,0 +1,18 @@ +Package: apt +Version: 0.7.25.3 +Architecture: i386 +Maintainer: APT Development Team +Installed-Size: 5244 +Replaces: libapt-pkg-dev (<< 0.3.7), libapt-pkg-doc (<< 0.3.7) +Provides: libapt-pkg-libc6.9-6-4.8 +Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt +Filename: apt.deb +Size: 0 +MD5sum: d41d8cd98f00b204e9800998ecf8427e +Description: Advanced front-end for dpkg + This is Debian's next generation front-end for the dpkg package manager. + It provides the apt-get utility and APT dselect method that provides a + simpler, safer way to install and upgrade packages. + . + APT features complete installation ordering, multiple source capability + and several other unique features, see the Users Guide in apt-doc. diff --git a/test/integration/Packages-releasefile-verification-new b/test/integration/Packages-releasefile-verification-new new file mode 100644 index 000000000..e3b2edf1f --- /dev/null +++ b/test/integration/Packages-releasefile-verification-new @@ -0,0 +1,21 @@ +Package: apt +Priority: important +Section: admin +Installed-Size: 5672 +Maintainer: APT Development Team +Architecture: i386 +Version: 0.8.0~pre1 +Replaces: manpages-pl (<< 20060617-3~) +Provides: libapt-pkg4.10 +Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt +Conflicts: python-apt (<< 0.7.93.2~) +Filename: apt.deb +Size: 0 +MD5sum: d41d8cd98f00b204e9800998ecf8427e +Description: Advanced front-end for dpkg + This is Debian's next generation front-end for the dpkg package manager. + It provides the apt-get utility and APT dselect method that provides a + simpler, safer way to install and upgrade packages. + . + APT features complete installation ordering, multiple source capability + and several other unique features, see the Users Guide in apt-doc. diff --git a/test/integration/framework b/test/integration/framework index 5d54e49b6..cb3fca35d 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -383,6 +383,7 @@ buildaptarchivefromfiles() { generatereleasefiles() { msgninfo "\tGenerate Release files… " + local DATE="${1:-now}" if [ -e aptarchive/dists ]; then for dir in $(find ./aptarchive/dists -mindepth 1 -maxdepth 1 -type d); do local CODENAME="$(echo "$dir" | cut -d'/' -f 4)" @@ -395,6 +396,11 @@ NotAutomatic: yes' $dir/Release else aptftparchive -qq release ./aptarchive | sed -e '/0 Release$/ d' > aptarchive/Release # remove the self reference fi + if [ "$DATE" != "now" ]; then + for release in $(find ./aptarchive -name 'Release'); do + touch -d "$1" $release + done + fi msgdone "info" } @@ -455,6 +461,7 @@ signreleasefiles() { done for RELEASE in $(find aptarchive/ -name Release); do gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE} + gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o "$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" $RELEASE done msgdone "info" } diff --git a/test/integration/marvinparanoid.pub b/test/integration/marvinparanoid.pub new file mode 100644 index 000000000..9a59c2e40 Binary files /dev/null and b/test/integration/marvinparanoid.pub differ diff --git a/test/integration/marvinparanoid.sec b/test/integration/marvinparanoid.sec new file mode 100644 index 000000000..ff54e8680 Binary files /dev/null and b/test/integration/marvinparanoid.sec differ diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 66792899a..c5379dca0 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -74,28 +74,24 @@ testoverfile() { setupcompressor "$1" createemptyfile 'en' - testaptgetupdate "Get:1 file: Release.gpg [] -Get:2 file: Release [] + testaptgetupdate "Get:1 file: InRelease [] Ign file:$(readlink -f aptarchive)/ Translation-en Reading package lists..." "empty file en.$COMPRESS over file" createemptyarchive 'en' - testaptgetupdate "Get:1 file: Release.gpg [] -Get:2 file: Release [] + testaptgetupdate "Get:1 file: InRelease [] Reading package lists..." "empty archive en.$COMPRESS over file" createemptyarchive 'Packages' # FIXME: Why omits the file transport the Packages Get line? #Get:3 file: Packages [] testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: Release.gpg [] -Get:2 file: Release [] +Get:1 file: InRelease [] Reading package lists..." "empty archive Packages.$COMPRESS over file" createemptyfile 'Packages' testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: Release.gpg [] -Get:2 file: Release [] +Get:1 file: InRelease [] Err file: Packages Undetermined Error W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS) Undetermined Error @@ -107,33 +103,29 @@ testoverhttp() { setupcompressor "$1" createemptyfile 'en' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Get:2 http://localhost/ Translation-en -Get:3 http://localhost Release [] +Get:3 http://localhost Packages [] Ign http://localhost/ Translation-en -Get:4 http://localhost Packages [] Reading package lists..." "empty file en.$COMPRESS over http" createemptyarchive 'en' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Get:2 http://localhost/ Translation-en [] -Get:3 http://localhost Release [] -Get:4 http://localhost Packages [] +Get:3 http://localhost Packages [] Reading package lists..." "empty archive en.$COMPRESS over http" createemptyarchive 'Packages' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Ign http://localhost/ Translation-en -Get:2 http://localhost Release [] -Get:3 http://localhost Packages [] +Get:2 http://localhost Packages [] Reading package lists..." "empty archive Packages.$COMPRESS over http" createemptyfile 'Packages' #FIXME: we should response with a good error message instead - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Ign http://localhost/ Translation-en -Get:2 http://localhost Release [] -Get:3 http://localhost Packages +Get:2 http://localhost Packages Err http://localhost Packages Undetermined Error W: Failed to fetch http://localhost:8080/Packages.$COMPRESS Undetermined Error diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification new file mode 100755 index 000000000..961c49895 --- /dev/null +++ b/test/integration/test-releasefile-verification @@ -0,0 +1,160 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +buildaptarchive +setupflataptarchive +changetowebserver + +prepare() { + local DATE="${2:-now}" + if [ "$DATE" = 'now' -a "$1" = "${PKGFILE}-new" ]; then + DATE='now + 6 days' + fi + for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do + touch -d 'now - 6 hours' $release + done + rm -rf rootdir/var/cache/apt/archives + rm -f rootdir/var/cache/apt/*.bin + cp $1 aptarchive/Packages + find aptarchive -name 'Release' -delete + cat aptarchive/Packages | gzip > aptarchive/Packages.gz + cat aptarchive/Packages | bzip2 > aptarchive/Packages.bz2 + cat aptarchive/Packages | lzma > aptarchive/Packages.lzma + generatereleasefiles "$DATE" +} + +installaptold() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5370 kB of additional disk space will be used. +Get:1 http://localhost/ apt 0.7.25.3 +Download complete and in download only mode' aptget install apt -dy +} + +installaptnew() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5808 kB of additional disk space will be used. +Get:1 http://localhost/ apt 0.8.0~pre1 +Download complete and in download only mode' aptget install apt -dy +} + +failaptold() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5370 kB of additional disk space will be used. +WARNING: The following packages cannot be authenticated! + apt +E: There are problems and -y was used without --force-yes' aptget install apt -dy +} + +failaptnew() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5808 kB of additional disk space will be used. +WARNING: The following packages cannot be authenticated! + apt +E: There are problems and -y was used without --force-yes' aptget install apt -dy +} + +# fake our downloadable file +touch aptarchive/apt.deb + +PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" + +runtest() { + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + prepare ${PKGFILE}-new + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Good warm archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt + installaptnew + + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Marvin Paranoid' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail + testequal "$(cat ${PKGFILE}) +" aptcache show apt + failaptold + + prepare ${PKGFILE}-new + # weborf doesn't support If-Range + for release in $(find rootdir/var/lib/apt/lists/partial/ -name '*Release'); do + rm $release + touch $release + done + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Bad warm archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt + installaptnew + + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + prepare ${PKGFILE}-new + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Good warm archive signed by' 'Marvin Paranoid' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold +} + +DELETEFILE="InRelease" +runtest +DELETEFILE="Release.gpg" +runtest -- cgit v1.2.3 From 3cb3fe76e90283a5b3fe7105e4854d4bac13bfeb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 20 Jan 2011 16:01:40 +0100 Subject: * ftparchive/writer.cc: - add config option to search for more patterns in release command --- debian/changelog | 4 +++- doc/apt-ftparchive.1.xml | 11 +++++++---- ftparchive/writer.cc | 26 +++++++++++++++----------- ftparchive/writer.h | 2 ++ 4 files changed, 27 insertions(+), 16 deletions(-) (limited to 'doc') diff --git a/debian/changelog b/debian/changelog index 59f516751..ca88901a1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -59,8 +59,10 @@ apt (0.8.11+wheezy) unstable; urgency=low can be used without being root (Closes: #393005, #592107) * apt-pkg/deb/deblistparser.cc: - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles + * ftparchive/writer.cc: + - add config option to search for more patterns in release command - -- David Kalnischkies Thu, 20 Jan 2011 14:52:32 +0100 + -- David Kalnischkies Thu, 20 Jan 2011 16:00:54 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/apt-ftparchive.1.xml b/doc/apt-ftparchive.1.xml index 852da8ad1..0090d21d9 100644 --- a/doc/apt-ftparchive.1.xml +++ b/doc/apt-ftparchive.1.xml @@ -113,10 +113,13 @@ release The release command generates a Release file from a - directory tree. It recursively searches the given directory for - Packages, Packages.gz, Packages.bz2, Sources, Sources.gz, - Sources.bz2, Release and md5sum.txt files. It then writes to - stdout a Release file containing an MD5 digest and SHA1 digest + directory tree. It recursively searches the given directory for uncompressed + Packages and Sources files and the ones + compressed with gzip, bzip2 or lzma + as well as Release and md5sum.txt files by default + (APT::FTPArchive::Release::Default-Patterns). Additional filename patterns + can be added by listing them in APT::FTPArchive::Release::Patterns. + It then writes to stdout a Release file containing a MD5, SHA1 and SHA256 digest for each file. Values for the additional metadata fields in the Release file are diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index d7d699ddc..55fac89bd 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -306,7 +306,7 @@ PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string c Output = stdout; SetExts(".deb .udeb"); DeLinkLimit = 0; - + // Process the command line options DoMD5 = _config->FindB("APT::FTPArchive::MD5",true); DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true); @@ -907,16 +907,20 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres /* */ ReleaseWriter::ReleaseWriter(string const &DB) { - AddPattern("Packages"); - AddPattern("Packages.gz"); - AddPattern("Packages.bz2"); - AddPattern("Packages.lzma"); - AddPattern("Sources"); - AddPattern("Sources.gz"); - AddPattern("Sources.bz2"); - AddPattern("Sources.lzma"); - AddPattern("Release"); - AddPattern("md5sum.txt"); + if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true) + { + AddPattern("Packages"); + AddPattern("Packages.gz"); + AddPattern("Packages.bz2"); + AddPattern("Packages.lzma"); + AddPattern("Sources"); + AddPattern("Sources.gz"); + AddPattern("Sources.bz2"); + AddPattern("Sources.lzma"); + AddPattern("Release"); + AddPattern("md5sum.txt"); + } + AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns")); Output = stdout; time_t const now = time(NULL); diff --git a/ftparchive/writer.h b/ftparchive/writer.h index 49d430c47..3796f79f6 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -69,6 +69,8 @@ class FTWScanner bool LoadFileList(string const &BaseDir,string const &File); void ClearPatterns() { Patterns.clear(); }; void AddPattern(string const &Pattern) { Patterns.push_back(Pattern); }; + void AddPattern(char const *Pattern) { Patterns.push_back(Pattern); }; + void AddPatterns(std::vector const &patterns) { Patterns.insert(Patterns.end(), patterns.begin(), patterns.end()); }; bool SetExts(string const &Vals); FTWScanner(string const &Arch = string()); -- cgit v1.2.3 From 5d88572318ed7e271101b1ae8f2cc139a1a3f705 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 26 Jan 2011 16:19:30 +0100 Subject: - change the internal handling of Extensions in pkgAcqIndex - add a special uncompressed compression type to prefer those files * methods/{gzip,bzip}.cc: - print a good error message if FileSize() is zero --- apt-pkg/acquire-item.cc | 114 ++++++++++++--------- apt-pkg/acquire-item.h | 9 +- apt-pkg/aptconfiguration.cc | 8 ++ debian/changelog | 6 +- doc/apt.conf.5.xml | 9 +- doc/examples/configure-index | 2 +- methods/bzip2.cc | 3 +- methods/gzip.cc | 3 +- .../test-bug-595691-empty-and-broken-archive-files | 9 +- 9 files changed, 96 insertions(+), 67 deletions(-) (limited to 'doc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2cd6ab359..a603a3d70 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -622,29 +622,61 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashString ExpectedHash, string comprExt) : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash) { + if(comprExt.empty() == true) + { + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + comprExt.append(*t).append(" "); + if (comprExt.empty() == false) + comprExt.erase(comprExt.end()-1); + } + CompressionExtension = comprExt; + + Init(URI, URIDesc, ShortDesc); +} +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) +{ + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + CompressionExtension = ""; + if (ExpectedHash.empty() == false) + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) + CompressionExtension.append(*t).append(" "); + } + else + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + CompressionExtension.append(*t).append(" "); + } + if (CompressionExtension.empty() == false) + CompressionExtension.erase(CompressionExtension.end()-1); + + Init(Target->URI, Target->Description, Target->ShortDesc); +} + /*}}}*/ +// AcqIndex::Init - defered Constructor /*{{{*/ +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Decompression = false; Erase = false; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - if(comprExt.empty()) - { - // autoselect the compression method - std::vector types = APT::Configuration::getCompressionTypes(); - if (types.empty() == true) - comprExt = "plain"; - else - comprExt = "." + types[0]; - } - CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt); - - Desc.URI = URI + CompressionExtension; + std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (comprExt == "uncompressed") + Desc.URI = URI; + else + Desc.URI = URI + '.' + comprExt; Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; - + QueueURI(Desc); } /*}}}*/ @@ -666,37 +698,18 @@ string pkgAcqIndex::Custom600Headers() /*}}}*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - std::vector types = APT::Configuration::getCompressionTypes(); - - for (std::vector::const_iterator t = types.begin(); - t != types.end(); t++) + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) { - // jump over all already tried compression types - const unsigned int nameLen = Desc.URI.size() - (*t).size(); - if(Desc.URI.substr(nameLen) != *t) - continue; - - // we want to try it with the next extension (and make sure to - // not skip over the end) - t++; - if (t == types.end()) - break; - - // queue new download - Desc.URI = Desc.URI.substr(0, nameLen) + *t; - new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc, - ExpectedHash, string(".").append(*t)); - - Status = StatDone; - Complete = false; - Dequeue(); + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); return; } // on decompression failure, remove bad versions in partial/ - if(Decompression && Erase) { + if (Decompression && Erase) { string s = _config->FindDir("Dir::State::lists") + "partial/"; - s += URItoFileName(RealURI); + s.append(URItoFileName(RealURI)); unlink(s.c_str()); } @@ -773,8 +786,8 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Status = StatError; ErrorText = "Method gave a blank filename"; } - - string compExt = flExtension(flNotDir(URI(Desc.URI).Path)); + + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); // The files timestamp matches if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { @@ -807,12 +820,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, // get the binary name for your used compression type decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); if(decompProg.empty() == false); - // flExtensions returns the full name if no extension is found - // this is why we have this complicated compare operation here - // FIMXE: add a new flJustExtension() that return "" if no - // extension is found and use that above so that it can - // be tested against "" - else if(compExt == flNotDir(URI(Desc.URI).Path)) + else if(compExt == "uncompressed") decompProg = "copy"; else { _error->Error("Unsupported extension: %s", compExt.c_str()); @@ -853,6 +861,15 @@ string pkgAcqIndexTrans::Custom600Headers() /* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) + { + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); + Status = StatIdle; + return; + } + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -862,7 +879,7 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Dequeue(); return; } - + Item::Failed(Message,Cnf); } /*}}}*/ @@ -1197,8 +1214,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHash); else - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); } } /*}}}*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 581761e32..92098e3d4 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -528,8 +528,8 @@ class pkgAcqIndex : public pkgAcquire::Item /** \brief The expected hashsum of the decompressed index file. */ HashString ExpectedHash; - /** \brief The compression-related file extension that is being - * added to the downloaded file (e.g., ".gz" or ".bz2"). + /** \brief The compression-related file extensions that are being + * added to the downloaded file one by one if first fails (e.g., "gz bz2"). */ string CompressionExtension; @@ -540,7 +540,7 @@ class pkgAcqIndex : public pkgAcquire::Item virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string Custom600Headers(); - virtual string DescURI() {return RealURI + CompressionExtension;}; + virtual string DescURI() {return Desc.URI;}; virtual string HashSum() {return ExpectedHash.toStr(); }; /** \brief Create a pkgAcqIndex. @@ -565,6 +565,9 @@ class pkgAcqIndex : public pkgAcquire::Item pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc, string ShortDesc, HashString ExpectedHash, string compressExt=""); + pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser); + void Init(string const &URI, string const &URIDesc, string const &ShortDesc); }; /*}}}*/ /** \brief An acquire item that is responsible for fetching a {{{ diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index 52f54073c..e97ebfed7 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -90,6 +90,14 @@ const Configuration::getCompressionTypes(bool const &Cached) { types.push_back(Types->Tag); } + // add the special "uncompressed" type + if (std::find(types.begin(), types.end(), "uncompressed") == types.end()) + { + string const uncompr = _config->FindFile("Dir::Bin::uncompressed", ""); + if (uncompr.empty() == true || FileExists(uncompr) == true) + types.push_back("uncompressed"); + } + return types; } /*}}}*/ diff --git a/debian/changelog b/debian/changelog index 2413b9c5b..7b125fc43 100644 --- a/debian/changelog +++ b/debian/changelog @@ -54,6 +54,8 @@ apt (0.8.11+wheezy) unstable; urgency=low * apt-pkg/acquire-item.cc: - don't uncompress downloaded pdiff files before feeding it to rred - try downloading clearsigned InRelease before trying Release.gpg + - change the internal handling of Extensions in pkgAcqIndex + - add a special uncompressed compression type to prefer those files * cmdline/apt-key: - don't set trustdb-name as non-root so 'list' and 'finger' can be used without being root (Closes: #393005, #592107) @@ -62,8 +64,10 @@ apt (0.8.11+wheezy) unstable; urgency=low * ftparchive/writer.cc: - add config option to search for more patterns in release command - include Index files by default in the Release file + * methods/{gzip,bzip}.cc: + - print a good error message if FileSize() is zero - -- David Kalnischkies Mon, 24 Jan 2011 15:36:50 +0100 + -- David Kalnischkies Wed, 26 Jan 2011 16:06:10 +0100 apt (0.8.10.3) unstable; urgency=low diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index a19d85dbc..a423dac24 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -442,12 +442,11 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; the bzip2 method (the inbuilt) setting is Dir::Bin::bzip2 "/bin/bzip2"; Note also that list entries specified on the command line will be added at the end of the list specified in the configuration files, but before the default entries. To prefer a type in this case - over the ones specified in in the configuration files you can set the option direct - not in list style. + over the ones specified in the configuration files you can set the option direct - not in list style. This will not override the defined list, it will only prefix the list with this type. - While it is possible to add an empty compression type to the order list, but APT in its current - version doesn't understand it correctly and will display many warnings about not downloaded files - - these warnings are most of the time false negatives. Future versions will maybe include a way to - really prefer uncompressed files to support the usage of local mirrors. + The special type uncompressed can be used to give uncompressed files a + preference, but note that most archives doesn't provide uncompressed files so this is mostly only + useable for local mirrors. GzipIndexes diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 6c078d75f..fd14d4dd7 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -281,7 +281,7 @@ Acquire lzma "lzma"; gz "gzip"; - Order { "gz"; "lzma"; "bz2"; }; + Order { "uncompressed"; "gz"; "lzma"; "bz2"; }; }; Languages diff --git a/methods/bzip2.cc b/methods/bzip2.cc index ccc3669a2..42932dded 100644 --- a/methods/bzip2.cc +++ b/methods/bzip2.cc @@ -56,9 +56,8 @@ bool Bzip2Method::Fetch(FetchItem *Itm) // Open the source and destination files FileFd From(Path,FileFd::ReadOnly); - // FIXME add an error message saying that empty files can't be valid archives if(From.FileSize() == 0) - return false; + return _error->Error(_("Empty files can't be valid archives")); int GzOut[2]; if (pipe(GzOut) < 0) diff --git a/methods/gzip.cc b/methods/gzip.cc index f1c76066e..fc4e1ecfd 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -48,9 +48,8 @@ bool GzipMethod::Fetch(FetchItem *Itm) // Open the source and destination files FileFd From(Path,FileFd::ReadOnlyGzip); - // FIXME add an error message saying that empty files can't be valid archives if(From.FileSize() == 0) - return false; + return _error->Error(_("Empty files can't be valid archives")); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index c5379dca0..398d0cd1b 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -56,6 +56,7 @@ setupcompressor() { lzma) COMPRESS="lzma";; esac echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; }; +Dir::Bin::uncompressed \"/does/not/exist\"; Dir::Bin::gzip \"/does/not/exist\"; Dir::Bin::bzip2 \"/does/not/exist\"; Dir::Bin::lzma \"/does/not/exist\";" > rootdir/etc/apt/apt.conf.d/00compressor @@ -93,8 +94,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over file" testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en Get:1 file: InRelease [] Err file: Packages - Undetermined Error -W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS) Undetermined Error + Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS) Empty files can't be valid archives E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file" } @@ -127,8 +128,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http" Ign http://localhost/ Translation-en Get:2 http://localhost Packages Err http://localhost Packages - Undetermined Error -W: Failed to fetch http://localhost:8080/Packages.$COMPRESS Undetermined Error + Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" } -- cgit v1.2.3