deal better with acquiring the same URI multiple times

This is an unlikely event for indexes and co, but it can happen quiet
easily e.g. for changelogs where you want to get the changelogs for
multiple binary package(version)s which happen to all be built from a
single source.

The interesting part is that the Acquire system actually detected this
already and set the item requesting the URI again to StatDone - expect
that this is hardly sufficient: an Item must be Complete=true as well
to be considered truely done and that is only the tip of the ::Done
handling iceberg. So instead of this StatDone hack we allow QItems to be
owned by multiple items and notify all owners about everything now,
so that for the point of each item they got it downloaded just for them.

2 files changed, 81 insertions, 1 deletions

diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times
new file mode 100755
index 000000000..d329a39cb
--- /dev/null
+++ b/test/integration/test-acquire-same-file-multiple-times
@@ -0,0 +1,80 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+filedown() {
+	msgtest 'Downloading the same URI twice over file' "$1"
+	testsuccess --nomsg apthelper download-file file:///$APTARCHIVE/foo ./downloaded/foo1 '' file:///$APTARCHIVE/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	#cat download.log
+	testsuccess cmp $TESTFILE ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	#testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+
+testrun() {
+	$1 'no partial'
+	cp $TESTFILE ./downloaded/foo1
+	$1 'complete partial 1'
+	cp $TESTFILE ./downloaded/foo2
+	$1 'complete partial 2'
+	cp $TESTFILE ./downloaded/foo1
+	cp $TESTFILE ./downloaded/foo2
+	$1 'complete partial 1+2'
+	dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null
+	$1 'partial partial 1'
+	dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null
+	$1 'partial partial 2'
+	dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null
+	dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null
+	$1 'partial partial 1+2'
+}
+testrun 'filedown'
+
+changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
+
+httpdown() {
+	msgtest 'Downloading the same URI to different files' 'twice over http'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+testrun 'httpdown'
+
+httpredirectdown() {
+	msgtest 'Redirect leads' 'first URI to the second URI'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '103%20Redirect' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+testrun 'httpredirectdown'
+
+httpsamedown() {
+	msgtest 'Downloading two files via the same URI to' 'the same file'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo1 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1
+}
+testrun 'httpsamedown'
diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download
index 6503bbd1c..fa0b65672 100755
--- a/test/integration/test-apt-get-download
+++ b/test/integration/test-apt-get-download
@@ -38,7 +38,7 @@ testdownload() {
 		APT="${APT}/${3}"
 	fi
 	msgtest "Test download of package file $1 with" "$APT"
-	testsuccess --nomsg aptget download ${APT}
+	testsuccess --nomsg aptget download ${APT} -o Debug::pkgAcquire::Worker=1 -o Debug::pkgAcquire::Auth=1
 	testsuccess test -f "$1"
 	rm -f "$1"
 }