Merge branch 'feature/methods'

10 files changed, 242 insertions, 42 deletions

diff --git a/test/integration/skip-method-http-socks-client b/test/integration/skip-method-http-socks-client
new file mode 100755
index 000000000..ee08f6d34
--- /dev/null
+++ b/test/integration/skip-method-http-socks-client
@@ -0,0 +1,116 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+
+# We don't do a real proxy here, we just look how the implementation
+# reacts to certain responses from a "proxy" provided by socat
+# Checks HTTP, but requesting https instead will check HTTPS (curl) which
+# uses different error messages through – also: https://github.com/curl/curl/issues/944
+
+# FIXME: Not run automatically as it uses a hardcoded port (5555)
+
+msgtest 'Check that everything is installed' 'socat'
+if dpkg-checkbuilddeps -d 'socat' /dev/null >/dev/null 2>&1; then
+	msgpass
+else
+	msgskip "$(command dpkg -l socat)"
+	exit
+fi
+
+runclient() {
+	# this doesn't need to be an actually reachable webserver for this test
+	# in fact, its better if it isn't.
+	rm -f index.html
+	apthelper download-file http://localhost:2903/ index.html \
+		-o Acquire::http::Proxy="socks5h://${1}localhost:5555" \
+		-o Acquire::http::Timeout=2 -o Debug::Acquire::http=1 > client.output 2>&1 || true
+}
+runserver() {
+	socat -x tcp-listen:5555,reuseaddr \
+		system:"echo -n '$*' | xxd -r -p; echo 'HTTP/1.1 200 OK'; echo 'Content-Length: 5'; echo 'Connection: close'; echo; echo 'HTML'" \
+		> server.output 2>&1 &
+}
+PROXY="socks5h://localhost:5555"
+
+msgmsg 'SOCKS does not run'
+runclient
+testsuccess grep 'Could not connect to localhost:5555' client.output
+
+msgmsg 'SOCKS greets back with wrong version'
+runserver '04 00'
+runclient
+testsuccess grep 'greets back with wrong version: 4' client.output
+
+msgmsg 'SOCKS tries GSSAPI auth we have not advertised'
+runserver '05 01'
+runclient
+testsuccess grep 'greets back having not found a common authorization method: 1' client.output
+
+msgmsg 'SOCKS tries user&pass auth we have not advertised'
+runserver '05 02'
+runclient
+testsuccess grep 'pass auth, but we had not offered it' client.output
+
+msgmsg 'SOCKS user:pass wrong version'
+runserver '05 02' '05 00'
+runclient 'user:pass@'
+testsuccess grep 'auth status response with wrong version: 5' client.output
+
+msgmsg 'SOCKS user:pass wrong auth'
+runserver '05 02' '01 01'
+runclient 'user:pass@'
+testsuccess grep 'reported authorization failure: username or password incorrect? (1)' client.output
+
+msgmsg 'SOCKS user:pass request not granted no hostname'
+runserver '05 02' '01 00' '05 01 00 03 00 1f 90'
+runclient 'user:pass@'
+testsuccess grep 'grant the connect to :8080 due to: general SOCKS server failure (1)' client.output
+
+msgmsg 'SOCKS user:pass request not granted with hostname'
+runserver '05 02' '01 00' '05 01 00 03 09 68 6f 73 74 6c 6f 63 61 6c 1f 90'
+runclient 'user:pass@'
+testsuccess grep 'grant the connect to hostlocal:8080 due to: general SOCKS server failure (1)' client.output
+
+msgmsg 'SOCKS user:pass request not granted ipv4'
+runserver '05 02' '01 00' '05 04 00 01 ac 10 fe 01 1f 90'
+runclient 'user:pass@'
+testsuccess grep 'grant the connect to 172.16.254.1:8080 due to: Host unreachable (4)' client.output
+
+msgmsg 'SOCKS user:pass request not granted ipv6'
+runserver '05 02' '01 00' '05 12 00 04 20 01 0d b8 ac 10 fe 00 00 00 00 00 00 00 00 00 1f 90'
+runclient 'user:pass@'
+testsuccess grep 'grant the connect to \[2001:0DB8:AC10:FE00:0000:0000:0000:0000\]:8080 due to: Unknown error (18)' client.output
+
+msgmsg 'SOCKS user:pass request granted ipv4'
+runserver '05 02' '01 00' '05 00 00 01 ac 10 fe 01 1f 90'
+runclient 'user:pass@'
+testequal "http: SOCKS proxy $PROXY connection established to 172.16.254.1:8080" head -n 1 client.output
+testfileequal index.html 'HTML'
+
+msgmsg 'SOCKS user:pass request granted ipv6'
+runserver '05 02' '01 00' '05 00 00 04 20 01 0d b8 ac 10 fe 00 00 00 00 00 00 00 00 00 1f 90'
+runclient 'user:pass@'
+testequal "http: SOCKS proxy $PROXY connection established to [2001:0DB8:AC10:FE00:0000:0000:0000:0000]:8080" head -n 1 client.output
+testfileequal index.html 'HTML'
+
+msgmsg 'SOCKS no auth no hostname'
+runserver '05 00 05 00 00 03 00 1f 90'
+runclient
+testequal "http: SOCKS proxy $PROXY connection established to :8080" head -n 1 client.output
+testfileequal index.html 'HTML'
+
+msgmsg 'SOCKS no auth with hostname'
+runserver '05 00 05 00 00 03 09 68 6f 73 74 6c 6f 63 61 6c 1f 90'
+runclient
+testequal "http: SOCKS proxy $PROXY connection established to hostlocal:8080" head -n 1 client.output
+testfileequal index.html 'HTML'
+
+msgmsg 'SOCKS user-only request granted ipv4'
+runserver '05 02' '01 00' '05 00 00 01 ac 10 fe 01 1f 90'
+runclient 'apt@'
+testequal "http: SOCKS proxy $PROXY connection established to 172.16.254.1:8080" head -n 1 client.output
+testfileequal index.html 'HTML'
diff --git a/test/integration/test-apt-https-no-redirect b/test/integration/test-apt-https-no-redirect
index b437ac1e6..d6c630d5f 100755
--- a/test/integration/test-apt-https-no-redirect
+++ b/test/integration/test-apt-https-no-redirect
@@ -11,7 +11,10 @@ insertpackage 'stable' 'apt' 'all' '1'
 setupaptarchive --no-update
 
 echo 'alright' > aptarchive/working
-changetohttpswebserver  -o "aptwebserver::redirect::replace::/redirectme/=http://localhost:${APTHTTPPORT}/"
+changetohttpswebserver
+webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/"
+webserverconfig 'aptwebserver::redirect::replace::/redirectme2/' "https://localhost:${APTHTTPSPORT}/"
+echo 'Dir::Bin::Methods::https+http "https";' > rootdir/etc/apt/apt.conf.d/99add-https-http-method
 
 msgtest 'download of a file works via' 'http'
 testsuccess --nomsg downloadfile "http://localhost:${APTHTTPPORT}/working" httpfile
@@ -20,9 +23,23 @@ testfileequal httpfile 'alright'
 msgtest 'download of a file works via' 'https'
 testsuccess --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/working" httpsfile
 testfileequal httpsfile 'alright'
+rm -f httpfile httpsfile
+
+msgtest 'download of http file works via' 'https+http'
+testsuccess --nomsg downloadfile "http://localhost:${APTHTTPPORT}/working" httpfile
+testfileequal httpfile 'alright'
+
+msgtest 'download of https file works via' 'https+http'
+testsuccess --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/working" httpsfile
+testfileequal httpsfile 'alright'
+rm -f httpfile httpsfile
 
 msgtest 'download of a file does not work if' 'https redirected to http'
 testfailure --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/redirectme/working" redirectfile
 
 msgtest 'libcurl has forbidden access in last request to' 'http resource'
-testsuccess --nomsg grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' rootdir/tmp/testfailure.output
+testsuccess --nomsg grep -q -E -- "Redirection from https to 'http://.*' is forbidden" rootdir/tmp/testfailure.output
+
+msgtest 'download of a file does work if' 'https+http redirected to https'
+testsuccess --nomsg downloadfile "https+http://localhost:${APTHTTPPORT}/redirectme2/working" redirectfile
+testfileequal redirectfile 'alright'
diff --git a/test/integration/test-apt-redirect-loop b/test/integration/test-apt-redirect-loop
new file mode 100755
index 000000000..b54f74276
--- /dev/null
+++ b/test/integration/test-apt-redirect-loop
@@ -0,0 +1,23 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "i386"
+
+insertpackage 'stable' 'apt' 'all' '1'
+setupaptarchive --no-update
+
+echo 'alright' > aptarchive/working
+changetohttpswebserver
+webserverconfig 'aptwebserver::redirect::replace::/redirectme3/' '/redirectme/'
+webserverconfig 'aptwebserver::redirect::replace::/redirectme2/' '/redirectme3/'
+webserverconfig 'aptwebserver::redirect::replace::/redirectme/' '/redirectme2/'
+
+testfailure apthelper download-file "http://localhost:${APTHTTPPORT}/redirectme/working" httpfile
+testsuccess grep 'Redirection loop encountered' rootdir/tmp/testfailure.output
+
+testfailure apthelper download-file "https://localhost:${APTHTTPSPORT}/redirectme/working" httpsfile
+testsuccess grep 'Redirection loop encountered' rootdir/tmp/testfailure.output
diff --git a/test/integration/test-apt-update-failure-propagation b/test/integration/test-apt-update-failure-propagation
index ec6bf4a48..1361b1b93 100755
--- a/test/integration/test-apt-update-failure-propagation
+++ b/test/integration/test-apt-update-failure-propagation
@@ -27,10 +27,11 @@ for FILE in rootdir/etc/apt/sources.list.d/*-sid-* ; do
 done
 
 pretest() {
+	msgmsg "$@"
 	rm -rf rootdir/var/lib/apt/lists
 	testsuccessequal 'N: Unable to locate package foo' aptcache policy foo
 }
-pretest
+pretest 'initialize test' 'update'
 testsuccess aptget update
 testsuccessequal "foo:
   Installed: (none)
@@ -41,7 +42,7 @@ testsuccessequal "foo:
      1 500
         500 https://localhost:${APTHTTPSPORT} stable/main all Packages" aptcache policy foo
 
-pretest
+pretest 'not found' 'release files'
 mv aptarchive/dists/stable aptarchive/dists/stable.good
 testfailuremsg "E: The repository 'https://localhost:${APTHTTPSPORT} stable Release' does not have a Release file.
 N: Updating from such a repository can't be done securely, and is therefore disabled by default.
@@ -76,7 +77,16 @@ posttest() {
 }
 posttest
 
-pretest
+pretest 'method disabled' 'https'
+echo 'Dir::Bin::Methods::https "false";' > rootdir/etc/apt/apt.conf.d/99disable-https
+testfailuremsg "E: The method 'https' is explicitly disabled via configuration.
+N: If you meant to use Tor remember to use tor+https instead of https.
+E: Failed to fetch https://localhost:${APTHTTPSPORT}/dists/stable/InRelease  
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update
+rm -f rootdir/etc/apt/apt.conf.d/99disable-https
+posttest
+
+pretest 'method not installed' 'https'
 rm "${NEWMETHODS}/https"
 testfailuremsg "E: The method driver ${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods/https could not be found.
 N: Is the package apt-transport-https installed?
@@ -84,13 +94,12 @@ E: Failed to fetch https://localhost:${APTHTTPSPORT}/dists/stable/InRelease
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update
 posttest
 
-ln -s "$OLDMETHODS/https" "$NEWMETHODS"
-pretest
+pretest 'https connection refused' 'doom port'
 for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do
 	# lets see how many testservers run also Doom
 	sed -i -e "s#:${APTHTTPSPORT}/#:666/#" "$FILE"
 done
-testwarning aptget update
+testwarning aptget update -o Dir::Bin::Methods::https="${OLDMETHODS}/https"
 testequalor2 "W: Failed to fetch https://localhost:666/dists/stable/InRelease  Failed to connect to localhost port 666: Connection refused
 W: Some index files failed to download. They have been ignored, or old ones used instead." "W: Failed to fetch https://localhost:666/dists/stable/InRelease  couldn't connect to host
 W: Some index files failed to download. They have been ignored, or old ones used instead." tail -n 2 rootdir/tmp/testwarning.output
diff --git a/test/integration/test-bug-623443-fail-on-bad-proxies b/test/integration/test-bug-623443-fail-on-bad-proxies
new file mode 100755
index 000000000..04542e0cd
--- /dev/null
+++ b/test/integration/test-bug-623443-fail-on-bad-proxies
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+
+setupaptarchive --no-update
+changetowebserver
+
+testsuccess apt update
+rm -rf rootdir/var/lib/apt/lists
+
+export http_proxy=enrico:password@proxy-cache.localnet:3128
+testfailure apt update
+unset http_proxy
+testsuccess grep 'Unsupported proxy configured' rootdir/tmp/testfailure.output
+
+changetohttpswebserver
+
+testsuccess apt update
+rm -rf rootdir/var/lib/apt/lists
+
+export http_proxy=enrico:password@proxy-cache.localnet:3128
+testfailure apt update
+unset http_proxy
+testsuccess grep 'Unsupported proxy configured' rootdir/tmp/testfailure.output
+
+
+
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 8f8216d04..690e8727e 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -38,29 +38,13 @@ cd - >/dev/null
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 
-# install a slowed down file: otherwise its to fast to reproduce combining
-NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
-OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
-rm "$NEWMETHODS"
-mkdir "$NEWMETHODS"
-backupIFS="$IFS"
-IFS="$(printf "\n\b")"
-for METH in $(find "$OLDMETHODS" -maxdepth 1 ! -type d); do
-	ln -s "$OLDMETHODS/$(basename "$METH")" "$NEWMETHODS"
-done
-IFS="$backupIFS"
-rm "$NEWMETHODS/https"
-
 cd downloaded
-testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
-N: Is the package apt-transport-https installed?" aptget download apt
+testfailureequal "E: The method 'https' is explicitly disabled via configuration.
+N: If you meant to use Tor remember to use tor+https instead of https." aptget download apt -o Dir::Bin::Methods::https=false
 testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
 
-# revert to all methods
-ln -s "$OLDMETHODS/https" "$NEWMETHODS"
-
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'
 sed -i -e "s#:${APTHTTPPORT}/redirectme#:${APTHTTPSPORT}/downgrademe#" -e 's# http:# https:#' rootdir/etc/apt/sources.list.d/*
-testfailure aptget update --allow-insecure-repositories
+testfailure aptget update --allow-insecure-repositories -o Acquire::https::Timeout=1
diff --git a/test/integration/test-different-methods-for-same-source b/test/integration/test-different-methods-for-same-source
index 6b3f796b1..791a84cb7 100755
--- a/test/integration/test-different-methods-for-same-source
+++ b/test/integration/test-different-methods-for-same-source
@@ -10,24 +10,26 @@ insertpackage 'stable' 'foo' 'all' '1'
 insertsource 'stable' 'foo' 'all' '1'
 setupaptarchive --no-update
 
-# install a slowed down file: otherwise its to fast to reproduce combining
-NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
-OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
-rm "$NEWMETHODS"
-mkdir "$NEWMETHODS"
-backupIFS="$IFS"
-IFS="$(printf "\n\b")"
-for METH in $(find "$OLDMETHODS" -maxdepth 1 ! -type d); do
-	ln -s "$OLDMETHODS/$(basename "$METH")" "$NEWMETHODS"
-done
-IFS="$backupIFS"
-ln -s "${OLDMETHODS}/http" "${NEWMETHODS}/http-ng"
-
 changetowebserver
+webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/"
+
+echo 'Dir::Bin::Methods::http-ng "http";' > rootdir/etc/apt/apt.conf.d/99add-http-ng-method
 sed -i -e 's# http:# http-ng:#' $(find rootdir/etc/apt/sources.list.d -name '*-deb-src.list')
 
-testsuccess apt update
+testsuccess apt update -o Debug::Acquire::http-ng=1
 cp rootdir/tmp/testsuccess.output update.log
 # all requests are folded into the first Release file
 testsuccess grep ' http-ng://' update.log
 testfailure grep ' http://' update.log
+# see if method-specific debug was enabled
+testsuccess grep '^Answer for: http-ng:' update.log
+
+rm -rf rootdir/var/lib/apt/lists
+sed -i -e "s#:${APTHTTPPORT}/#:${APTHTTPPORT}/redirectme#" rootdir/etc/apt/sources.list.d/*
+testsuccess apt update -o Debug::Acquire::http-ng=1
+cp rootdir/tmp/testsuccess.output update.log
+# all requests are folded into the first Release file
+testsuccess grep ' http-ng://' update.log
+testfailure grep '^[^L].* http://' update.log
+# see if method-specific debug was enabled
+testsuccess grep '^Answer for: http-ng:' update.log
diff --git a/test/integration/test-method-connect b/test/integration/test-method-connect
new file mode 100755
index 000000000..b35f96dc3
--- /dev/null
+++ b/test/integration/test-method-connect
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+
+cd downloaded
+testfailureequal 'Err:1 http://vwakviie2ienjx6t.onion/
+  Direct connection to .onion domains is blocked by default. If you meant to use Tor remember to use tor+http instead of http.
+E: Failed to fetch http://vwakviie2ienjx6t.onion/  Direct connection to .onion domains is blocked by default. If you meant to use Tor remember to use tor+http instead of http.
+E: Download Failed' apthelper download-file 'http://vwakviie2ienjx6t.onion/' ftp.debian.org.html
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index e2d2743b3..1c5d120d8 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -146,3 +146,6 @@ serverconfigs "http://localhost:${APTHTTPPORT}"
 changetohttpswebserver
 
 serverconfigs "https://localhost:${APTHTTPSPORT}"
+
+webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "https://localhost:${APTHTTPSPORT}/"
+serverconfigs "http://localhost:${APTHTTPPORT}/redirectme"
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 91528389b..39d847203 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -317,7 +317,7 @@ Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::Worker "true";
 Debug::Acquire::http "true";
 Debug::pkgAcquire "true";
-Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
+Debug::Acquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
 testcase() {
 	testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"