keep compressed indexes in a low-cost format

Downloading and storing are two different operations were different
compression types can be preferred. For downloading we provide the
choice via Acquire::CompressionTypes::Order as there is a choice to
be made between download size and speed – and limited by whats available
in the repository.

Storage on the other hand has all compressions currently supported by
apt available and to reduce runtime of tools accessing these files the
compression type should be a low-cost format in terms of decompression.

apt traditionally stores its indexes uncompressed on disk, but has
options to keep them compressed. Now that apt downloads additional files
we also deal with files which simply can't be stored uncompressed as
they are just too big (like Contents for apt-file). Traditionally they
are downloaded in a low-cost format (gz) as repositories do not provide
other formats, but there might be even lower-cost formats and for
download we could introduce higher-cost in the repositories.

Downloading an entire index potentially requires recompression to
another format, so an update takes potentially longer – but big files
are usually updated via pdiffs which has to de- and re-compress anyhow
and does it on the fly anyhow, so there is no extra time needed and in
general it seems to be benefitial to invest the time in update to save
time later on file access.

6 files changed, 27 insertions, 20 deletions

diff --git a/test/integration/framework b/test/integration/framework
index e2d545ec2..c97500ffd 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1826,7 +1826,7 @@ listcurrentlistsdirectory() {
 }
 forallsupportedcompressors() {
 	for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
-		if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = 'lz4' ]; then continue; fi
+		if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
 		"$@" "$COMP"
 	done
 }
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index e9bbdbce6..bf7850a7a 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -10,6 +10,7 @@ configarchitecture 'amd64'
 # note that in --print-uri we talk about .xz because that is the default.
 # This doesn't mean it is actually attempt to download it.
 configcompression '.' 'gz'
+LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
 
@@ -101,14 +102,14 @@ Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s
 Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]
 Reading package lists..." aptget update
 
-testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
-testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" find rootdir/var/lib/apt/lists -name '*Contents-all*'
-testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
-testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" 'aptarchive/dists/unstable/main/Contents-amd64.gz'
-testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" 'aptarchive/dists/unstable/main/Contents-all.gz'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
+testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
 
-rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz
-rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 
 msgmsg "Compressed Contents file"
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index f175ca355..d433baeb5 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -24,15 +24,15 @@ runtest() {
     rm -rf rootdir/var/lib/apt/lists/
 
     local TEST="test${1:-success}"
-    $TEST aptget update $APTOPT
+    $TEST aptget update $APTOPT -o Debug::pkgAcquire::Worker=1
     if [ "$1" = 'failure' ]; then
 	# accept the outdated Release file so we can check Hit behaviour
 	"test${2:-success}" aptget update -o Acquire::Min-ValidTime=9999999 $APTOPT
     fi
     listcurrentlistsdirectory > listsdir.lst
-    testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst
-    testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst
-    testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Packages\(\.[0-9a-z]\+\)\?$' listsdir.lst
+    testsuccess grep '_Sources\(\.[0-9a-z]\+\)\?$' listsdir.lst
+    testsuccess grep '_Translation-en\(\.[0-9a-z]\+\)\?$' listsdir.lst
 
     # ensure no leftovers in partial
     testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth
index 2ececc4f6..ef1e4eca7 100755
--- a/test/integration/test-apt-update-unauth
+++ b/test/integration/test-apt-update-unauth
@@ -12,6 +12,7 @@ umask 022
 
 setupenvironment
 configarchitecture "i386"
+configcompression '.' 'xz'
 
 insertpackage 'unstable' 'foo' 'i386' '1.0'
 insertsource 'unstable' 'foo' 'any' '1.0'
@@ -45,7 +46,7 @@ runtest() {
     listcurrentlistsdirectory > lists.before
 
     # update and ensure all is reverted on the hashsum failure
-    testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
+    testfailure aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
 
     # ensure we have before what we have after
     msgtest 'Check rollback on going from' 'unauth -> auth'
@@ -53,10 +54,13 @@ runtest() {
     if cmp lists.before lists.after; then
         msgpass
     else
+	echo >&2
 	echo >&2 '### Output of previous apt-get update ###'
-	cat >&2 rootdir/tmp/testfailure.output
+	cat >&2 rootdir/tmp/testfailure.output || true
 	echo >&2 '### Changes in the lists-directory: ###'
-	diff -u >&2 lists.before lists.after
+	diff -u >&2 lists.before lists.after || true
+	echo >&2 '### Contents of the lists-directory: ###'
+	ls -l rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists/partial || true
 	msgfail
     fi
 
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index b97e1244b..573a86511 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
 setupenvironment
 configcompression '.' $(aptconfig dump APT::Compressor --format '%t %v%n' | sed -n 's#^Extension \.\(.*\)$#\1#p')
 configarchitecture 'i386'
+LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'testpkg' 'i386' '1.0'
 
@@ -35,10 +36,10 @@ testrun() {
 		! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
 		! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
-		test -e rootdir/var/lib/apt/lists/*i386_Packages.${COMPRESS} || F=1
-		test -e rootdir/var/lib/apt/lists/*all_Packages.${COMPRESS} || F=1
-		test -e rootdir/var/lib/apt/lists/*_Sources.${COMPRESS} || F=1
-		test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1
+		test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
+		test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
+		test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
+		test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
 		# there is no point in trying pdiff if we have compressed indexes
 		# as we can't patch compressed files (well, we can, but what is the point?)
 		! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 547242cad..32240da4e 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
 setupenvironment
 configarchitecture 'i386'
 confighashes 'SHA1' 'SHA256'
+LOWCOSTEXT='lz4'
 
 buildaptarchive
 setupflataptarchive
@@ -274,5 +275,5 @@ testcase() {
 }
 aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages"; }
 testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=false
-aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.gz"; }
+aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.$LOWCOSTEXT"; }
 testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=true