fail earlier if server answers with too much data

We tend to operate on rather large static files, which means we usually
get Content-Length information from the server. If we combine this
information with the filesize we are expecting (factoring in pipelining)
we can avoid reading a bunch of data we are ending up rejecting anyhow
by just closing the connection saving bandwidth and time both for the
server as well as the client.

5 files changed, 25 insertions, 17 deletions

diff --git a/methods/curl.cc b/methods/curl.cc
index dfdd3b06b..a19318098 100644
--- a/methods/curl.cc
+++ b/methods/curl.cc
@@ -139,8 +139,8 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
       if (TotalWritten > me->https->Queue->MaximumSize)
       {
 	 me->https->SetFailReason("MaximumSizeExceeded");
-	 _error->Error("Writing more data than expected (%llu > %llu)",
-	       TotalWritten, me->https->Queue->MaximumSize);
+	 _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+		       TotalWritten, me->https->Queue->MaximumSize);
 	 return 0;
       }
    }
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 9bfe72bc6..dd97458d0 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -940,8 +940,8 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
       if (MaximumSize > 0 && To.Tell() > MaximumSize)
       {
          Owner->SetFailReason("MaximumSizeExceeded");
-         return _error->Error("Writing more data than expected (%llu > %llu)",
-                              To.Tell(), MaximumSize);
+	 return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+			      To.Tell(), MaximumSize);
       }
    }
 
diff --git a/methods/http.cc b/methods/http.cc
index 9f01d15b3..9425145dd 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -267,6 +267,7 @@ bool CircleBuf::Write(string &Data)
    OutP += LeftWrite();
    return true;
 }
+									/*}}}*/
 // CircleBuf::Stats - Print out stats information			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -282,11 +283,12 @@ void CircleBuf::Stats()
    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 }
 									/*}}}*/
-CircleBuf::~CircleBuf()
+CircleBuf::~CircleBuf()							/*{{{*/
 {
    delete [] Buf;
    delete Hash;
 }
+									/*}}}*/
 
 // UnwrapHTTPConnect - Does the HTTP CONNECT handshake			/*{{{*/
 // ---------------------------------------------------------------------
@@ -590,7 +592,15 @@ bool HttpServerState::RunData(RequestState &Req)
       if (Req.JunkSize != 0)
 	 In.Limit(Req.JunkSize);
       else if (Req.DownloadSize != 0)
+      {
+	 if (Req.MaximumSize != 0 && Req.DownloadSize > Req.MaximumSize)
+	 {
+	    Owner->SetFailReason("MaximumSizeExceeded");
+	    return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+				 Req.DownloadSize, Req.MaximumSize);
+	 }
 	 In.Limit(Req.DownloadSize);
+      }
       else if (Persistent == false)
 	 In.Limit(-1);
       
@@ -827,8 +837,8 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req)
    if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize)
    {
       Owner->SetFailReason("MaximumSizeExceeded");
-      return _error->Error("Writing more data than expected (%llu > %llu)",
-                           Req.File.Tell(), Req.MaximumSize);
+      return _error->Error(_("File is larger than expected (%llu > %llu). Mirror sync in progress?"),
+			   Req.File.Tell(), Req.MaximumSize);
    }
 
    // Handle commands from APT
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 710d05d18..5c73a2396 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -21,7 +21,7 @@ test_inreleasetoobig() {
 	testsuccess aptget update -o Apt::Get::List-Cleanup=0  -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0
 	msgtest 'Check that the max write warning is triggered'
 	cp rootdir/tmp/testsuccess.output update.output
-	testsuccess --nomsg grep -q 'Writing more data than expected' update.output
+	testsuccess --nomsg grep -q 'File is larger than expected' update.output
 	rm -f update.output
 	# ensure the failed InRelease file got renamed
 	testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED
@@ -39,7 +39,7 @@ test_packagestoobig() {
 		touch -d '+1hour' "$pkg"
 	done
 	NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages.gz)"
-	testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz  Writing more data than expected ($NEW_SIZE > $SIZE)
+	testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz  File is larger than expected ($NEW_SIZE > $SIZE). Mirror sync in progress?
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::Transaction=0
 	testsuccess ls rootdir/var/lib/apt/lists/partial/*Packages*.FAILED
 	testfailure test -e rootdir/var/lib/apt/lists/partial/Old.FAILED
@@ -51,29 +51,27 @@ methodtest() {
 	webserverconfig 'aptwebserver::support::last-modified' 'false' "$1"  # curl is clever and sees hits here also
 
 	msgmsg 'Test with' "$1" 'and clean start'
-	rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+	rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 	# normal update works fine
 	testsuccess aptget update
 	touch rootdir/var/lib/apt/lists/partial/Old.FAILED
 	mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
-
 	# starting fresh works
 	test_inreleasetoobig "$1"
 	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
 	cp -a aptarchive/dists.good aptarchive/dists
 	test_packagestoobig "$1"
-	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
-	cp -a aptarchive/dists.good aptarchive/dists
 
 	msgmsg 'Test with' "$1" 'and existing old data'
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
 	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
 	test_inreleasetoobig "$1"
 	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
 	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
 	cp -a aptarchive/dists.good aptarchive/dists
 	test_packagestoobig "$1"
-	rm -rf aptarchive/dists
-	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 changetowebserver
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 2636c3140..14e5a20b8 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -216,7 +216,7 @@ testmismatch() {
 Need to get 6 B of source archives.
 Get:1 http://localhost:${APTHTTPPORT}  $1 1.0 (dsc) [2 B]
 Err:1 http://localhost:${APTHTTPPORT}  $1 1.0 (dsc)
-  Writing more data than expected (3 > 2)
+  File is larger than expected (3 > 2). Mirror sync in progress?
   Hashes of expected file:
    - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a
    - Filesize:2 [weak]
@@ -230,7 +230,7 @@ Err:2 http://localhost:${APTHTTPPORT}  $1 1.0 (tar)
    - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb
    - Filesize:3 [weak]
   Last modification reported: $(lastmodification "aptarchive/${1}_1.0.dsc")
-E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc  Writing more data than expected (3 > 2)
+E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc  File is larger than expected (3 > 2). Mirror sync in progress?
    Hashes of expected file:
     - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a
     - Filesize:2 [weak]