diff options
author | David Kalnischkies <kalnischkies@gmail.com> | 2011-09-13 17:46:48 +0200 |
---|---|---|
committer | David Kalnischkies <kalnischkies@gmail.com> | 2011-09-13 17:46:48 +0200 |
commit | 650faab01603caac04494d54cf6b10a65c00ea13 (patch) | |
tree | d89d9ec876c195d8f757e1351858ea7c200dd269 | |
parent | ea54214002c09eeb4dd498d97a564471ec9993c5 (diff) | |
download | apt-650faab01603caac04494d54cf6b10a65c00ea13.tar.gz |
Support large files in the complete toolset. Indexes of this
size are pretty unlikely for now, but we need it for deb
packages which could become bigger than 4GB now (LP: #815895)
52 files changed, 270 insertions, 215 deletions
diff --git a/apt-inst/contrib/arfile.h b/apt-inst/contrib/arfile.h index 7f6c68302..e2063cd71 100644 --- a/apt-inst/contrib/arfile.h +++ b/apt-inst/contrib/arfile.h @@ -54,7 +54,7 @@ struct ARArchive::Member unsigned long UID; unsigned long GID; unsigned long Mode; - unsigned long Size; + unsigned long long Size; // Location of the data. unsigned long Start; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 366879a57..3bb977e14 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -258,9 +258,9 @@ bool pkgAcquire::Worker::RunMessages() CurrentItem = Itm; CurrentSize = 0; - TotalSize = atoi(LookupTag(Message,"Size","0").c_str()); - ResumePoint = atoi(LookupTag(Message,"Resume-Point","0").c_str()); - Itm->Owner->Start(Message,atoi(LookupTag(Message,"Size","0").c_str())); + TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10); + ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10); + Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10)); // Display update before completion if (Log != 0 && Log->MorePulses == true) @@ -289,7 +289,7 @@ bool pkgAcquire::Worker::RunMessages() Log->Pulse(Owner->GetOwner()); OwnerQ->ItemDone(Itm); - unsigned long long const ServerSize = atoll(LookupTag(Message,"Size","0").c_str()); + unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10); if (TotalSize != 0 && ServerSize != TotalSize) _error->Warning("Size of file %s is not what the server reported %s %llu", Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize); diff --git a/apt-pkg/contrib/crc-16.cc b/apt-pkg/contrib/crc-16.cc index 26ea1ba28..4058821f9 100644 --- a/apt-pkg/contrib/crc-16.cc +++ b/apt-pkg/contrib/crc-16.cc @@ -65,7 +65,7 @@ static unsigned short const crc16_table[256] = /* Recompute the FCS with one more character appended. */ #define CalcFCS(fcs, c) (((fcs) >> 8) ^ crc16_table[((fcs) ^ (c)) & 0xff]) unsigned short AddCRC16(unsigned short fcs, void const *Buf, - unsigned long len) + unsigned long long len) { unsigned char const *buf = (unsigned char const *)Buf; while (len--) diff --git a/apt-pkg/contrib/crc-16.h b/apt-pkg/contrib/crc-16.h index f30678bac..702de40b2 100644 --- a/apt-pkg/contrib/crc-16.h +++ b/apt-pkg/contrib/crc-16.h @@ -12,6 +12,6 @@ #define INIT_FCS 0xffff unsigned short AddCRC16(unsigned short fcs, void const *buf, - unsigned long len); + unsigned long long len); #endif diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 690e2403c..2e62846d9 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -133,10 +133,10 @@ bool CopyFile(FileFd &From,FileFd &To) // Buffered copy between fds SPtrArray<unsigned char> Buf = new unsigned char[64000]; - unsigned long Size = From.Size(); + unsigned long long Size = From.Size(); while (Size != 0) { - unsigned long ToRead = Size; + unsigned long long ToRead = Size; if (Size > 64000) ToRead = 64000; @@ -800,7 +800,7 @@ FileFd::~FileFd() // --------------------------------------------------------------------- /* We are carefull to handle interruption by a signal while reading gracefully. */ -bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual) +bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) { int Res; errno = 0; @@ -839,13 +839,13 @@ bool FileFd::Read(void *To,unsigned long Size,unsigned long *Actual) } Flags |= Fail; - return _error->Error(_("read, still have %lu to read but none left"),Size); + return _error->Error(_("read, still have %llu to read but none left"), Size); } /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Write(const void *From,unsigned long Size) +bool FileFd::Write(const void *From,unsigned long long Size) { int Res; errno = 0; @@ -872,13 +872,13 @@ bool FileFd::Write(const void *From,unsigned long Size) return true; Flags |= Fail; - return _error->Error(_("write, still have %lu to write but couldn't"),Size); + return _error->Error(_("write, still have %llu to write but couldn't"), Size); } /*}}}*/ // FileFd::Seek - Seek in the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Seek(unsigned long To) +bool FileFd::Seek(unsigned long long To) { int res; if (gz) @@ -888,7 +888,7 @@ bool FileFd::Seek(unsigned long To) if (res != (signed)To) { Flags |= Fail; - return _error->Error("Unable to seek to %lu",To); + return _error->Error("Unable to seek to %llu", To); } return true; @@ -897,7 +897,7 @@ bool FileFd::Seek(unsigned long To) // FileFd::Skip - Seek in the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Skip(unsigned long Over) +bool FileFd::Skip(unsigned long long Over) { int res; if (gz) @@ -907,7 +907,7 @@ bool FileFd::Skip(unsigned long Over) if (res < 0) { Flags |= Fail; - return _error->Error("Unable to seek ahead %lu",Over); + return _error->Error("Unable to seek ahead %llu",Over); } return true; @@ -916,7 +916,7 @@ bool FileFd::Skip(unsigned long Over) // FileFd::Truncate - Truncate the file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileFd::Truncate(unsigned long To) +bool FileFd::Truncate(unsigned long long To) { if (gz) { @@ -926,7 +926,7 @@ bool FileFd::Truncate(unsigned long To) if (ftruncate(iFd,To) != 0) { Flags |= Fail; - return _error->Error("Unable to truncate to %lu",To); + return _error->Error("Unable to truncate to %llu",To); } return true; @@ -935,7 +935,7 @@ bool FileFd::Truncate(unsigned long To) // FileFd::Tell - Current seek position /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long FileFd::Tell() +unsigned long long FileFd::Tell() { off_t Res; if (gz) @@ -950,7 +950,7 @@ unsigned long FileFd::Tell() // FileFd::FileSize - Return the size of the file /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long FileFd::FileSize() +unsigned long long FileFd::FileSize() { struct stat Buf; @@ -962,9 +962,9 @@ unsigned long FileFd::FileSize() // FileFd::Size - Return the size of the content in the file /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long FileFd::Size() +unsigned long long FileFd::Size() { - unsigned long size = FileSize(); + unsigned long long size = FileSize(); // only check gzsize if we are actually a gzip file, just checking for // "gz" is not sufficient as uncompressed files will be opened with @@ -974,6 +974,7 @@ unsigned long FileFd::Size() /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do * this ourselves; the original (uncompressed) file size is the last 32 * bits of the file */ + // FIXME: Size for gz-files is limited by 32bit… no largefile support off_t orig_pos = lseek(iFd, 0, SEEK_CUR); if (lseek(iFd, -4, SEEK_END) < 0) return _error->Errno("lseek","Unable to seek to end of gzipped file"); diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index cde288ad2..0f2dd4194 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -49,21 +49,36 @@ class FileFd enum OpenMode {ReadOnly,WriteEmpty,WriteExists,WriteAny,WriteTemp,ReadOnlyGzip, WriteAtomic}; - inline bool Read(void *To,unsigned long Size,bool AllowEof) + inline bool Read(void *To,unsigned long long Size,bool AllowEof) { - unsigned long Jnk; + unsigned long long Jnk; if (AllowEof) return Read(To,Size,&Jnk); return Read(To,Size); } - bool Read(void *To,unsigned long Size,unsigned long *Actual = 0); - bool Write(const void *From,unsigned long Size); - bool Seek(unsigned long To); - bool Skip(unsigned long To); - bool Truncate(unsigned long To); - unsigned long Tell(); - unsigned long Size(); - unsigned long FileSize(); + bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0); + bool Write(const void *From,unsigned long long Size); + bool Seek(unsigned long long To); + bool Skip(unsigned long long To); + bool Truncate(unsigned long long To); + unsigned long long Tell(); + unsigned long long Size(); + unsigned long long FileSize(); + + /* You want to use 'unsigned long long' if you are talking about a file + to be able to support large files (>2 or >4 GB) properly. + This shouldn't happen all to often for the indexes, but deb's might be… + And as the auto-conversation converts a 'unsigned long *' to a 'bool' + instead of 'unsigned long long *' we need to provide this explicitely - + otherwise applications magically start to fail… */ + __deprecated bool Read(void *To,unsigned long long Size,unsigned long *Actual) + { + unsigned long long R; + bool const T = Read(To, Size, &R); + *Actual = R; + return T; + } + bool Open(string FileName,OpenMode Mode,unsigned long Perms = 0666); bool OpenDescriptor(int Fd, OpenMode Mode, bool AutoClose=false); bool Close(); diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 9c251e89f..fd76bf229 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -108,18 +108,18 @@ string HashString::toStr() const // Hashes::AddFD - Add the contents of the FD /*{{{*/ // --------------------------------------------------------------------- /* */ -bool Hashes::AddFD(int const Fd,unsigned long Size, bool const addMD5, +bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5, bool const addSHA1, bool const addSHA256, bool const addSHA512) { unsigned char Buf[64*64]; - int Res = 0; + ssize_t Res = 0; int ToEOF = (Size == 0); while (Size != 0 || ToEOF) { - unsigned n = sizeof(Buf); - if (!ToEOF) n = min(Size,(unsigned long)n); + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = min(Size, n); Res = read(Fd,Buf,n); - if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read + if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read return false; if (ToEOF && Res == 0) // EOF break; diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index e702fcca2..40c2ad064 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -62,14 +62,14 @@ class Hashes SHA256Summation SHA256; SHA512Summation SHA512; - inline bool Add(const unsigned char *Data,unsigned long Size) + inline bool Add(const unsigned char *Data,unsigned long long Size) { return MD5.Add(Data,Size) && SHA1.Add(Data,Size) && SHA256.Add(Data,Size) && SHA512.Add(Data,Size); }; inline bool Add(const char *Data) {return Add((unsigned char *)Data,strlen(Data));}; - inline bool AddFD(int const Fd,unsigned long Size = 0) + inline bool AddFD(int const Fd,unsigned long long Size = 0) { return AddFD(Fd, Size, true, true, true, true); }; - bool AddFD(int const Fd, unsigned long Size, bool const addMD5, + bool AddFD(int const Fd, unsigned long long Size, bool const addMD5, bool const addSHA1, bool const addSHA256, bool const addSHA512); inline bool Add(const unsigned char *Beg,const unsigned char *End) {return Add(Beg,End-Beg);}; diff --git a/apt-pkg/contrib/hashsum.cc b/apt-pkg/contrib/hashsum.cc index 28f711176..0edcbb364 100644 --- a/apt-pkg/contrib/hashsum.cc +++ b/apt-pkg/contrib/hashsum.cc @@ -7,16 +7,16 @@ // Summation::AddFD - Add content of file into the checksum /*{{{*/ // --------------------------------------------------------------------- /* */ -bool SummationImplementation::AddFD(int const Fd, unsigned long Size) { +bool SummationImplementation::AddFD(int const Fd, unsigned long long Size) { unsigned char Buf[64 * 64]; - int Res = 0; + ssize_t Res = 0; int ToEOF = (Size == 0); while (Size != 0 || ToEOF) { - unsigned n = sizeof(Buf); - if (!ToEOF) n = min(Size,(unsigned long)n); + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = min(Size, n); Res = read(Fd, Buf, n); - if (Res < 0 || (!ToEOF && (unsigned) Res != n)) // error, or short read + if (Res < 0 || (!ToEOF && Res != (ssize_t) n)) // error, or short read return false; if (ToEOF && Res == 0) // EOF break; diff --git a/apt-pkg/contrib/hashsum_template.h b/apt-pkg/contrib/hashsum_template.h index 85d94c2af..9157754e3 100644 --- a/apt-pkg/contrib/hashsum_template.h +++ b/apt-pkg/contrib/hashsum_template.h @@ -87,8 +87,8 @@ class HashSumValue class SummationImplementation { public: - virtual bool Add(const unsigned char *inbuf, unsigned long inlen) = 0; - inline bool Add(const char *inbuf, unsigned long const inlen) + virtual bool Add(const unsigned char *inbuf, unsigned long long inlen) = 0; + inline bool Add(const char *inbuf, unsigned long long const inlen) { return Add((unsigned char *)inbuf, inlen); }; inline bool Add(const unsigned char *Data) @@ -101,7 +101,7 @@ class SummationImplementation inline bool Add(const char *Beg, const char *End) { return Add((const unsigned char *)Beg, End - Beg); }; - bool AddFD(int Fd, unsigned long Size = 0); + bool AddFD(int Fd, unsigned long long Size = 0); }; #endif diff --git a/apt-pkg/contrib/md5.cc b/apt-pkg/contrib/md5.cc index b53c4fbd2..4351aeb22 100644 --- a/apt-pkg/contrib/md5.cc +++ b/apt-pkg/contrib/md5.cc @@ -187,7 +187,7 @@ MD5Summation::MD5Summation() // MD5Summation::Add - 'Add' a data set to the hash /*{{{*/ // --------------------------------------------------------------------- /* */ -bool MD5Summation::Add(const unsigned char *data,unsigned long len) +bool MD5Summation::Add(const unsigned char *data,unsigned long long len) { if (Done == true) return false; diff --git a/apt-pkg/contrib/md5.h b/apt-pkg/contrib/md5.h index e76428325..305cdb20d 100644 --- a/apt-pkg/contrib/md5.h +++ b/apt-pkg/contrib/md5.h @@ -45,7 +45,7 @@ class MD5Summation : public SummationImplementation public: - bool Add(const unsigned char *inbuf, unsigned long inlen); + bool Add(const unsigned char *inbuf, unsigned long long inlen); using SummationImplementation::Add; MD5SumValue Result(); diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index 3cd87eda4..a110a7019 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -95,7 +95,7 @@ bool MMap::Map(FileFd &Fd) return false; } else - return _error->Errno("mmap",_("Couldn't make mmap of %lu bytes"), + return _error->Errno("mmap",_("Couldn't make mmap of %llu bytes"), iSize); } @@ -166,7 +166,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) return true; #ifdef _POSIX_SYNCHRONIZED_IO - unsigned long PSize = sysconf(_SC_PAGESIZE); + unsigned long long PSize = sysconf(_SC_PAGESIZE); if ((Flags & ReadOnly) != ReadOnly) { if (SyncToFd != 0) @@ -177,7 +177,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop) } else { - if (msync((char *)Base+(int)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0) + if (msync((char *)Base+(unsigned long long)(Start/PSize)*PSize,Stop - Start,MS_SYNC) < 0) return _error->Errno("msync", _("Unable to synchronize mmap")); } } @@ -197,7 +197,7 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Work if (_error->PendingError() == true) return; - unsigned long EndOfFile = Fd->Size(); + unsigned long long EndOfFile = Fd->Size(); if (EndOfFile > WorkSpace) WorkSpace = EndOfFile; else if(WorkSpace > 0) @@ -285,7 +285,7 @@ DynamicMMap::~DynamicMMap() return; } - unsigned long EndOfFile = iSize; + unsigned long long EndOfFile = iSize; iSize = WorkSpace; Close(false); if(ftruncate(Fd->Fd(),EndOfFile) < 0) @@ -295,9 +295,9 @@ DynamicMMap::~DynamicMMap() // DynamicMMap::RawAllocate - Allocate a raw chunk of unaligned space /*{{{*/ // --------------------------------------------------------------------- /* This allocates a block of memory aligned to the given size */ -unsigned long DynamicMMap::RawAllocate(unsigned long Size,unsigned long Aln) +unsigned long DynamicMMap::RawAllocate(unsigned long long Size,unsigned long Aln) { - unsigned long Result = iSize; + unsigned long long Result = iSize; if (Aln != 0) Result += Aln - (iSize%Aln); @@ -412,7 +412,7 @@ bool DynamicMMap::Grow() { if (GrowFactor <= 0) return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user.")); - unsigned long const newSize = WorkSpace + GrowFactor; + unsigned long long const newSize = WorkSpace + GrowFactor; if(Fd != 0) { Fd->Seek(newSize - 1); diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h index 2bf2c1540..e0ff8db95 100644 --- a/apt-pkg/contrib/mmap.h +++ b/apt-pkg/contrib/mmap.h @@ -41,7 +41,7 @@ class MMap protected: unsigned long Flags; - unsigned long iSize; + unsigned long long iSize; void *Base; // In case mmap can not be used, we keep a dup of the file @@ -60,8 +60,8 @@ class MMap // Simple accessors inline operator void *() {return Base;}; inline void *Data() {return Base;}; - inline unsigned long Size() {return iSize;}; - inline void AddSize(unsigned long const size) {iSize += size;}; + inline unsigned long long Size() {return iSize;}; + inline void AddSize(unsigned long long const size) {iSize += size;}; inline bool validData() const { return Base != (void *)-1 && Base != 0; }; // File manipulators @@ -99,7 +99,7 @@ class DynamicMMap : public MMap public: // Allocation - unsigned long RawAllocate(unsigned long Size,unsigned long Aln = 0); + unsigned long RawAllocate(unsigned long long Size,unsigned long Aln = 0); unsigned long Allocate(unsigned long ItemSize); unsigned long WriteString(const char *String,unsigned long Len = (unsigned long)-1); inline unsigned long WriteString(const string &S) {return WriteString(S.c_str(),S.length());}; diff --git a/apt-pkg/contrib/progress.cc b/apt-pkg/contrib/progress.cc index 6cd6134d3..317048845 100644 --- a/apt-pkg/contrib/progress.cc +++ b/apt-pkg/contrib/progress.cc @@ -37,7 +37,7 @@ OpProgress::OpProgress() : Current(0), Total(0), Size(0), SubTotal(1), /* Current is the Base Overall progress in units of Total. Cur is the sub progress in units of SubTotal. Size is a scaling factor that says what percent of Total SubTotal is. */ -void OpProgress::Progress(unsigned long Cur) +void OpProgress::Progress(unsigned long long Cur) { if (Total == 0 || Size == 0 || SubTotal == 0) Percent = 0; @@ -49,8 +49,8 @@ void OpProgress::Progress(unsigned long Cur) // OpProgress::OverallProgress - Set the overall progress /*{{{*/ // --------------------------------------------------------------------- /* */ -void OpProgress::OverallProgress(unsigned long Current, unsigned long Total, - unsigned long Size,const string &Op) +void OpProgress::OverallProgress(unsigned long long Current, unsigned long long Total, + unsigned long long Size,const string &Op) { this->Current = Current; this->Total = Total; @@ -67,7 +67,7 @@ void OpProgress::OverallProgress(unsigned long Current, unsigned long Total, // OpProgress::SubProgress - Set the sub progress state /*{{{*/ // --------------------------------------------------------------------- /* */ -void OpProgress::SubProgress(unsigned long SubTotal,const string &Op, +void OpProgress::SubProgress(unsigned long long SubTotal,const string &Op, float const Percent) { this->SubTotal = SubTotal; diff --git a/apt-pkg/contrib/progress.h b/apt-pkg/contrib/progress.h index 3a914d17f..5344323f6 100644 --- a/apt-pkg/contrib/progress.h +++ b/apt-pkg/contrib/progress.h @@ -30,10 +30,10 @@ using std::string; class Configuration; class OpProgress { - unsigned long Current; - unsigned long Total; - unsigned long Size; - unsigned long SubTotal; + unsigned long long Current; + unsigned long long Total; + unsigned long long Size; + unsigned long long SubTotal; float LastPercent; // Change reduction code @@ -54,10 +54,10 @@ class OpProgress public: - void Progress(unsigned long Current); - void SubProgress(unsigned long SubTotal, const string &Op = "", float const Percent = -1); - void OverallProgress(unsigned long Current,unsigned long Total, - unsigned long Size,const string &Op); + void Progress(unsigned long long Current); + void SubProgress(unsigned long long SubTotal, const string &Op = "", float const Percent = -1); + void OverallProgress(unsigned long long Current,unsigned long long Total, + unsigned long long Size,const string &Op); virtual void Done() {}; OpProgress(); diff --git a/apt-pkg/contrib/sha1.cc b/apt-pkg/contrib/sha1.cc index 9416895ac..31c576611 100644 --- a/apt-pkg/contrib/sha1.cc +++ b/apt-pkg/contrib/sha1.cc @@ -243,7 +243,7 @@ SHA1SumValue SHA1Summation::Result() // SHA1Summation::Add - Adds content of buffer into the checksum /*{{{*/ // --------------------------------------------------------------------- /* May not be called after Result() is called */ -bool SHA1Summation::Add(const unsigned char *data,unsigned long len) +bool SHA1Summation::Add(const unsigned char *data,unsigned long long len) { if (Done) return false; diff --git a/apt-pkg/contrib/sha1.h b/apt-pkg/contrib/sha1.h index 2701fc67e..916faec1b 100644 --- a/apt-pkg/contrib/sha1.h +++ b/apt-pkg/contrib/sha1.h @@ -34,7 +34,7 @@ class SHA1Summation : public SummationImplementation bool Done; public: - bool Add(const unsigned char *inbuf, unsigned long inlen); + bool Add(const unsigned char *inbuf, unsigned long long inlen); using SummationImplementation::Add; SHA1SumValue Result(); diff --git a/apt-pkg/contrib/sha2.h b/apt-pkg/contrib/sha2.h index 386225889..51c921dbd 100644 --- a/apt-pkg/contrib/sha2.h +++ b/apt-pkg/contrib/sha2.h @@ -30,7 +30,7 @@ class SHA2SummationBase : public SummationImplementation protected: bool Done; public: - bool Add(const unsigned char *inbuf, unsigned long len) = 0; + bool Add(const unsigned char *inbuf, unsigned long long len) = 0; void Result(); }; @@ -41,7 +41,7 @@ class SHA256Summation : public SHA2SummationBase unsigned char Sum[32]; public: - bool Add(const unsigned char *inbuf, unsigned long len) + bool Add(const unsigned char *inbuf, unsigned long long len) { if (Done) return false; @@ -73,7 +73,7 @@ class SHA512Summation : public SHA2SummationBase unsigned char Sum[64]; public: - bool Add(const unsigned char *inbuf, unsigned long len) + bool Add(const unsigned char *inbuf, unsigned long long len) { if (Done) return false; diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index 6586ef17b..04226f1b4 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -970,6 +970,34 @@ bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base) return true; } /*}}}*/ +// StrToNum - Convert a fixed length string to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the crazy fixed length string headers in + tar and ar files. */ +bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base) +{ + char S[30]; + if (Len >= sizeof(S)) + return false; + memcpy(S,Str,Len); + S[Len] = 0; + + // All spaces is a zero + Res = 0; + unsigned I; + for (I = 0; S[I] == ' '; I++); + if (S[I] == 0) + return true; + + char *End; + Res = strtoull(S,&End,Base); + if (End == S) + return false; + + return true; +} + /*}}}*/ + // Base256ToNum - Convert a fixed length binary to a number /*{{{*/ // --------------------------------------------------------------------- /* This is used in decoding the 256bit encoded fixed length fields in diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 89cbf0370..b32198f25 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -52,6 +52,7 @@ string LookupTag(const string &Message,const char *Tag,const char *Default = 0); int StringToBool(const string &Text,int Default = -1); bool ReadMessages(int Fd, vector<string> &List); bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base = 0); +bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base = 0); bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len); bool Hex2Num(const string &Str,unsigned char *Num,unsigned int Length); bool TokSplitString(char Tok,char *Input,char **List, diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index 0fbd77fd8..8dde7c173 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -171,14 +171,14 @@ bool pkgDepCache::readStateFile(OpProgress *Prog) /*{{{*/ string const state = _config->FindFile("Dir::State::extended_states"); if(RealFileExists(state)) { state_file.Open(state, FileFd::ReadOnly); - int const file_size = state_file.Size(); + off_t const file_size = state_file.Size(); if(Prog != NULL) Prog->OverallProgress(0, file_size, 1, _("Reading state information")); pkgTagFile tagfile(&state_file); pkgTagSection section; - int amt = 0; + off_t amt = 0; bool const debug_autoremove = _config->FindB("Debug::pkgAutoRemove",false); while(tagfile.Step(section)) { string const pkgname = section.FindS("Package"); diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index dcba78dce..f52ecbbcb 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -29,6 +29,7 @@ #include <sys/types.h> #include <fcntl.h> #include <stdio.h> +#include <stdlib.h> #include "indexcopy.h" #include <apti18n.h> @@ -55,7 +56,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, bool Debug = _config->FindB("Debug::aptcdrom",false); // Prepare the progress indicator - unsigned long TotalSize = 0; + off_t TotalSize = 0; for (vector<string>::iterator I = List.begin(); I != List.end(); I++) { struct stat Buf; @@ -66,14 +67,14 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, TotalSize += Buf.st_size; } - unsigned long CurrentSize = 0; + off_t CurrentSize = 0; unsigned int NotFound = 0; unsigned int WrongSize = 0; unsigned int Packages = 0; for (vector<string>::iterator I = List.begin(); I != List.end(); I++) { string OrigPath = string(*I,CDROM.length()); - unsigned long FileSize = 0; + off_t FileSize = 0; // Open the package file FileFd Pkg; @@ -166,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, if(Progress) Progress->Progress(Parser.Offset()); string File; - unsigned long Size; + unsigned long long Size; if (GetFile(File,Size) == false) { fclose(TargetFl); @@ -221,7 +222,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, } // Size match - if ((unsigned)Buf.st_size != Size) + if ((unsigned long long)Buf.st_size != Size) { if (Debug == true) clog << "Wrong Size: " << File << endl; @@ -455,7 +456,7 @@ bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth) // PackageCopy::GetFile - Get the file information from the section /*{{{*/ // --------------------------------------------------------------------- /* */ -bool PackageCopy::GetFile(string &File,unsigned long &Size) +bool PackageCopy::GetFile(string &File,unsigned long long &Size) { File = Section->FindS("Filename"); Size = Section->FindI("Size"); @@ -481,7 +482,7 @@ bool PackageCopy::RewriteEntry(FILE *Target,string File) // SourceCopy::GetFile - Get the file information from the section /*{{{*/ // --------------------------------------------------------------------- /* */ -bool SourceCopy::GetFile(string &File,unsigned long &Size) +bool SourceCopy::GetFile(string &File,unsigned long long &Size) { string Files = Section->FindS("Files"); if (Files.empty() == true) @@ -504,7 +505,7 @@ bool SourceCopy::GetFile(string &File,unsigned long &Size) return _error->Error("Error parsing file record"); // Parse the size and append the directory - Size = atoi(sSize.c_str()); + Size = strtoull(sSize.c_str(), NULL, 10); File = Base + File; return true; } @@ -787,7 +788,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ bool Debug = _config->FindB("Debug::aptcdrom",false); // Prepare the progress indicator - unsigned long TotalSize = 0; + off_t TotalSize = 0; for (vector<string>::iterator I = List.begin(); I != List.end(); I++) { struct stat Buf; @@ -798,14 +799,14 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ TotalSize += Buf.st_size; } - unsigned long CurrentSize = 0; + off_t CurrentSize = 0; unsigned int NotFound = 0; unsigned int WrongSize = 0; unsigned int Packages = 0; for (vector<string>::iterator I = List.begin(); I != List.end(); I++) { string OrigPath = string(*I,CDROM.length()); - unsigned long FileSize = 0; + off_t FileSize = 0; // Open the package file FileFd Pkg; diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index 277fb561c..60c90dd4a 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -37,7 +37,7 @@ class IndexCopy /*{{{*/ bool ReconstructChop(unsigned long &Chop,string Dir,string File); void ConvertToSourceList(string CD,string &Path); bool GrabFirst(string Path,string &To,unsigned int Depth); - virtual bool GetFile(string &Filename,unsigned long &Size) = 0; + virtual bool GetFile(string &Filename,unsigned long long &Size) = 0; virtual bool RewriteEntry(FILE *Target,string File) = 0; virtual const char *GetFileName() = 0; virtual const char *Type() = 0; @@ -53,7 +53,7 @@ class PackageCopy : public IndexCopy /*{{{*/ { protected: - virtual bool GetFile(string &Filename,unsigned long &Size); + virtual bool GetFile(string &Filename,unsigned long long &Size); virtual bool RewriteEntry(FILE *Target,string File); virtual const char *GetFileName() {return "Packages";}; virtual const char *Type() {return "Package";}; @@ -64,7 +64,7 @@ class SourceCopy : public IndexCopy /*{{{*/ { protected: - virtual bool GetFile(string &Filename,unsigned long &Size); + virtual bool GetFile(string &Filename,unsigned long long &Size); virtual bool RewriteEntry(FILE *Target,string File); virtual const char *GetFileName() {return "Sources";}; virtual const char *Type() {return "Source";}; diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 00f520c4f..932640764 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -80,7 +80,7 @@ bool indexRecords::Load(const string Filename) /*{{{*/ string Name; string Hash; - size_t Size; + unsigned long long Size; while (Start < End) { if (!parseSumData(Start, End, Name, Hash, Size)) @@ -147,7 +147,7 @@ vector<string> indexRecords::MetaKeys() /*{{{*/ } /*}}}*/ bool indexRecords::parseSumData(const char *&Start, const char *End, /*{{{*/ - string &Name, string &Hash, size_t &Size) + string &Name, string &Hash, unsigned long long &Size) { Name = ""; Hash = ""; @@ -184,7 +184,7 @@ bool indexRecords::parseSumData(const char *&Start, const char *End, /*{{{*/ if (EntryEnd == End) return false; - Size = strtol (Start, NULL, 10); + Size = strtoull (Start, NULL, 10); /* Skip over intermediate blanks */ Start = EntryEnd; diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h index 5b532c1a5..0f933b93c 100644 --- a/apt-pkg/indexrecords.h +++ b/apt-pkg/indexrecords.h @@ -17,7 +17,7 @@ class indexRecords { bool parseSumData(const char *&Start, const char *End, string &Name, - string &Hash, size_t &Size); + string &Hash, unsigned long long &Size); public: struct checkSum; string ErrorText; @@ -53,7 +53,7 @@ struct indexRecords::checkSum { string MetaKeyFilename; HashString Hash; - size_t Size; + unsigned long long Size; }; #endif diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index 78e39e577..840454e18 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -19,7 +19,6 @@ #include <apt-pkg/pkgcache.h> -#include <apt-pkg/fileutl.h> #include <vector> class pkgRecords /*{{{*/ diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 3b491fcd2..418e6bed8 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -29,7 +29,7 @@ using std::string; class pkgTagFilePrivate { public: - pkgTagFilePrivate(FileFd *pFd, unsigned long Size) : Fd(*pFd), Size(Size) + pkgTagFilePrivate(FileFd *pFd, unsigned long long Size) : Fd(*pFd), Size(Size) { } FileFd &Fd; @@ -37,14 +37,14 @@ public: char *Start; char *End; bool Done; - unsigned long iOffset; - unsigned long Size; + unsigned long long iOffset; + unsigned long long Size; }; // TagFile::pkgTagFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long Size) +pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size) { d = new pkgTagFilePrivate(pFd, Size); @@ -86,7 +86,7 @@ unsigned long pkgTagFile::Offset() bool pkgTagFile::Resize() { char *tmp; - unsigned long EndSize = d->End - d->Start; + unsigned long long EndSize = d->End - d->Start; // fail is the buffer grows too big if(d->Size > 1024*1024+1) @@ -138,8 +138,8 @@ bool pkgTagFile::Step(pkgTagSection &Tag) then fills the rest from the file */ bool pkgTagFile::Fill() { - unsigned long EndSize = d->End - d->Start; - unsigned long Actual = 0; + unsigned long long EndSize = d->End - d->Start; + unsigned long long Actual = 0; memmove(d->Buffer,d->Start,EndSize); d->Start = d->Buffer; @@ -180,12 +180,12 @@ bool pkgTagFile::Fill() // --------------------------------------------------------------------- /* This jumps to a pre-recorded file location and reads the record that is there */ -bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long Offset) +bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset) { // We are within a buffer space of the next hit.. if (Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset) { - unsigned long Dist = Offset - d->iOffset; + unsigned long long Dist = Offset - d->iOffset; d->Start += Dist; d->iOffset += Dist; return Step(Tag); diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 23f5c57e6..87d070d28 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -94,9 +94,9 @@ class pkgTagFile bool Step(pkgTagSection &Section); unsigned long Offset(); - bool Jump(pkgTagSection &Tag,unsigned long Offset); + bool Jump(pkgTagSection &Tag,unsigned long long Offset); - pkgTagFile(FileFd *F,unsigned long Size = 32*1024); + pkgTagFile(FileFd *F,unsigned long long Size = 32*1024); virtual ~pkgTagFile(); }; diff --git a/buildlib/config.h.in b/buildlib/config.h.in index b3609a700..256911231 100644 --- a/buildlib/config.h.in +++ b/buildlib/config.h.in @@ -33,6 +33,9 @@ /* If there is no socklen_t, define this for the netdb shim */ #undef NEED_SOCKLEN_T_DEFINE +/* Define to the size of the filesize containing structures */ +#undef _FILE_OFFSET_BITS + /* Define the arch name string */ #undef COMMON_ARCH diff --git a/cmdline/acqprogress.cc b/cmdline/acqprogress.cc index fd80ea126..1ccb08804 100644 --- a/cmdline/acqprogress.cc +++ b/cmdline/acqprogress.cc @@ -163,7 +163,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner) ScreenWidth = sizeof(Buffer)-1; // Put in the percent done - sprintf(S,"%ld%%",long(double((CurrentBytes + CurrentItems)*100.0)/double(TotalBytes+TotalItems))); + sprintf(S,"%.0f%%",((CurrentBytes + CurrentItems)*100.0)/(TotalBytes+TotalItems)); bool Shown = false; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; @@ -214,11 +214,11 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner) if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false) { if (Mode == Short) - snprintf(S,End-S," %lu%%", - long(double(I->CurrentSize*100.0)/double(I->TotalSize))); + snprintf(S,End-S," %.0f%%", + (I->CurrentSize*100.0)/I->TotalSize); else - snprintf(S,End-S,"/%sB %lu%%",SizeToStr(I->TotalSize).c_str(), - long(double(I->CurrentSize*100.0)/double(I->TotalSize))); + snprintf(S,End-S,"/%sB %.0f%%",SizeToStr(I->TotalSize).c_str(), + (I->CurrentSize*100.0)/I->TotalSize); } S += strlen(S); snprintf(S,End-S,"]"); @@ -238,7 +238,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner) if (CurrentCPS != 0) { char Tmp[300]; - unsigned long ETA = (unsigned long)((TotalBytes - CurrentBytes)/CurrentCPS); + unsigned long long ETA = (TotalBytes - CurrentBytes)/CurrentCPS; sprintf(Tmp," %sB/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str()); unsigned int Len = strlen(Buffer); unsigned int LenT = strlen(Tmp); diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 1bf4cf6f9..9b9c0f0b0 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -27,9 +27,6 @@ // Include Files /*{{{*/ #include <config.h> -#define _LARGEFILE_SOURCE -#define _LARGEFILE64_SOURCE - #include <apt-pkg/aptconfiguration.h> #include <apt-pkg/error.h> #include <apt-pkg/cmndline.h> @@ -68,9 +65,6 @@ #include <sstream> #include <apti18n.h> - -#define statfs statfs64 -#define statvfs statvfs64 /*}}}*/ #define RAMFS_MAGIC 0x858458f6 diff --git a/configure.in b/configure.in index 3dde2fe47..965bcb5d1 100644 --- a/configure.in +++ b/configure.in @@ -31,6 +31,10 @@ AC_CHECK_TOOL_PREFIX AC_PROG_CC AC_ISC_POSIX +dnl check for large file support and enable it if possible +dnl do this early as other stuff might depend on it +AC_SYS_LARGEFILE + dnl Check for other programs AC_PROG_CXX AC_PROG_CPP @@ -151,7 +155,7 @@ if test "$cross_compiling" = "yes" -a "x$archline" = "x"; then AC_MSG_ERROR(When cross compiling, architecture must be present in sizetable) fi AC_C_BIGENDIAN - + dnl We do not need this if we have inttypes! HAVE_C9X=yes if test x"$apt_cv_c9x_ints" = x"no"; then diff --git a/debian/changelog b/debian/changelog index ce2a6a8fc..dd91d3aa8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +apt (0.8.16~exp3+nmu1) experimental; urgency=low + + [ David Kalnischkies ] + * Support large files in the complete toolset. Indexes of this + size are pretty unlikely for now, but we need it for deb + packages which could become bigger than 4GB now (LP: #815895) + + -- David Kalnischkies <kalnischkies@gmail.com> Tue, 13 Sep 2011 17:44:00 +0200 + apt (0.8.16~exp3) experimental; urgency=low [ David Kalnischkies ] diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 6ad8ac572..5721fe409 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -206,7 +206,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) Packages.Output = 0; // Just in case // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false) { c0out << endl; @@ -292,7 +292,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) Sources.Output = 0; // Just in case // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false) { c0out << endl; @@ -363,11 +363,11 @@ bool PackageMap::GenContents(Configuration &Setup, if (_error->PendingError() == true) return false; - unsigned long Size = Head.Size(); + unsigned long long Size = Head.Size(); unsigned char Buf[4096]; while (Size != 0) { - unsigned long ToRead = Size; + unsigned long long ToRead = Size; if (Size > sizeof(Buf)) ToRead = sizeof(Buf); @@ -401,7 +401,7 @@ bool PackageMap::GenContents(Configuration &Setup, Contents.Finish(); // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false || _error->PendingError() == true) { c0out << endl; diff --git a/ftparchive/cachedb.h b/ftparchive/cachedb.h index 15e796325..6bd5aa36c 100644 --- a/ftparchive/cachedb.h +++ b/ftparchive/cachedb.h @@ -81,7 +81,7 @@ class CacheDB { uint32_t Flags; uint32_t mtime; - uint32_t FileSize; + uint64_t FileSize; uint8_t MD5[16]; uint8_t SHA1[20]; uint8_t SHA256[32]; @@ -114,7 +114,7 @@ class CacheDB double SHA512Bytes; unsigned long Packages; unsigned long Misses; - unsigned long DeLinkBytes; + unsigned long long DeLinkBytes; inline void Add(const Stats &S) { Bytes += S.Bytes; @@ -133,7 +133,7 @@ class CacheDB inline bool DBFailed() {return Dbp != 0 && DBLoaded == false;}; inline bool Loaded() {return DBLoaded == true;}; - inline off_t GetFileSize(void) {return CurStat.FileSize;} + inline unsigned long long GetFileSize(void) {return CurStat.FileSize;} bool SetFile(string const &FileName,struct stat St,FileFd *Fd); bool GetFileInfo(string const &FileName, bool const &DoControl, bool const &DoContents, bool const &GenContentsOnly, diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc index adb590ed1..65d8dd1ce 100644 --- a/ftparchive/contents.cc +++ b/ftparchive/contents.cc @@ -349,7 +349,7 @@ bool ContentsExtract::DoItem(Item &Itm,int &Fd) // ContentsExtract::TakeContents - Load the contents data /*{{{*/ // --------------------------------------------------------------------- /* */ -bool ContentsExtract::TakeContents(const void *NewData,unsigned long Length) +bool ContentsExtract::TakeContents(const void *NewData,unsigned long long Length) { if (Length == 0) { diff --git a/ftparchive/contents.h b/ftparchive/contents.h index 5b5092b66..f549ce17f 100644 --- a/ftparchive/contents.h +++ b/ftparchive/contents.h @@ -71,15 +71,15 @@ class ContentsExtract : public pkgDirStream // The Data Block char *Data; - unsigned long MaxSize; - unsigned long CurSize; + unsigned long long MaxSize; + unsigned long long CurSize; void AddData(const char *Text); bool Read(debDebFile &Deb); virtual bool DoItem(Item &Itm,int &Fd); void Reset() {CurSize = 0;}; - bool TakeContents(const void *Data,unsigned long Length); + bool TakeContents(const void *Data,unsigned long long Length); void Add(GenContents &Contents,string const &Package); ContentsExtract() : Data(0), MaxSize(0), CurSize(0) {}; diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc index 04560f4ab..a80d6a34d 100644 --- a/ftparchive/multicompress.cc +++ b/ftparchive/multicompress.cc @@ -215,7 +215,7 @@ bool MultiCompress::Die() // MultiCompress::Finalize - Finish up writing /*{{{*/ // --------------------------------------------------------------------- /* This is only necessary for statistics reporting. */ -bool MultiCompress::Finalize(unsigned long &OutSize) +bool MultiCompress::Finalize(unsigned long long &OutSize) { OutSize = 0; if (Input == 0 || Die() == false) @@ -383,7 +383,7 @@ bool MultiCompress::Child(int const &FD) stash a hash of the data to use later. */ SetNonBlock(FD,false); unsigned char Buffer[32*1024]; - unsigned long FileSize = 0; + unsigned long long FileSize = 0; MD5Summation MD5; while (1) { @@ -445,7 +445,7 @@ bool MultiCompress::Child(int const &FD) // Compute the hash MD5Summation OldMD5; - unsigned long NewFileSize = 0; + unsigned long long NewFileSize = 0; while (1) { int Res = read(CompFd,Buffer,sizeof(Buffer)); diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h index 19dede174..4839af47d 100644 --- a/ftparchive/multicompress.h +++ b/ftparchive/multicompress.h @@ -54,7 +54,7 @@ class MultiCompress FILE *Input; unsigned long UpdateMTime; - bool Finalize(unsigned long &OutSize); + bool Finalize(unsigned long long &OutSize); bool OpenOld(int &Fd,pid_t &Proc); bool CloseOld(int Fd,pid_t Proc); static bool GetStat(string const &Output,string const &Compress,struct stat &St); diff --git a/ftparchive/override.cc b/ftparchive/override.cc index 045a8b113..a101fa6d1 100644 --- a/ftparchive/override.cc +++ b/ftparchive/override.cc @@ -34,7 +34,7 @@ bool Override::ReadOverride(string const &File,bool const &Source) return _error->Errno("fopen",_("Unable to open %s"),File.c_str()); char Line[500]; - unsigned long Counter = 0; + unsigned long long Counter = 0; while (fgets(Line,sizeof(Line),F) != 0) { Counter++; @@ -57,7 +57,7 @@ bool Override::ReadOverride(string const &File,bool const &Source) for (; isspace(*End) == 0 && *End != 0; End++); if (*End == 0) { - _error->Warning(_("Malformed override %s line %lu #1"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #1"),File.c_str(), Counter); continue; } @@ -71,7 +71,7 @@ bool Override::ReadOverride(string const &File,bool const &Source) for (; isspace(*End) == 0 && *End != 0; End++); if (*End == 0) { - _error->Warning(_("Malformed override %s line %lu #2"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #2"),File.c_str(), Counter); continue; } @@ -85,7 +85,7 @@ bool Override::ReadOverride(string const &File,bool const &Source) for (; isspace(*End) == 0 && *End != 0; End++); if (*End == 0) { - _error->Warning(_("Malformed override %s line %lu #3"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #3"),File.c_str(), Counter); continue; } @@ -142,7 +142,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source) return _error->Errno("fopen",_("Unable to open %s"),File.c_str()); char Line[500]; - unsigned long Counter = 0; + unsigned long long Counter = 0; while (fgets(Line,sizeof(Line),F) != 0) { Counter++; @@ -163,7 +163,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source) for (; isspace(*End) == 0 && *End != 0; End++); if (*End == 0) { - _error->Warning(_("Malformed override %s line %lu #1"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #1"),File.c_str(), Counter); continue; } @@ -175,7 +175,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source) for (; isspace(*End) == 0 && *End != 0; End++); if (*End == 0) { - _error->Warning(_("Malformed override %s line %lu #2"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #2"),File.c_str(), Counter); continue; } @@ -188,7 +188,7 @@ bool Override::ReadExtraOverride(string const &File,bool const &Source) for (; isspace(*(End-1)) && End > Value; End--); if (End == Value) { - _error->Warning(_("Malformed override %s line %lu #3"),File.c_str(), + _error->Warning(_("Malformed override %s line %llu #3"),File.c_str(), Counter); continue; } diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 65269c6ae..c4e700b59 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -248,8 +248,8 @@ bool FTWScanner::LoadFileList(string const &Dir, string const &File) // --------------------------------------------------------------------- /* */ bool FTWScanner::Delink(string &FileName,const char *OriginalPath, - unsigned long &DeLinkBytes, - off_t const &FileSize) + unsigned long long &DeLinkBytes, + unsigned long long const &FileSize) { // See if this isn't an internaly prefix'd file name. if (InternalPrefix.empty() == false && @@ -379,7 +379,7 @@ bool PackagesWriter::DoPackage(string FileName) return false; } - off_t FileSize = Db.GetFileSize(); + unsigned long long FileSize = Db.GetFileSize(); if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false) return false; @@ -415,7 +415,7 @@ bool PackagesWriter::DoPackage(string FileName) } char Size[40]; - sprintf(Size,"%lu", (unsigned long) FileSize); + sprintf(Size,"%llu", (unsigned long long) FileSize); // Strip the DirStrip prefix from the FileName and add the PathPrefix string NewFileName; @@ -612,7 +612,7 @@ bool SourcesWriter::DoPackage(string FileName) if (St.st_size > 128*1024) return _error->Error("DSC file '%s' is too large!",FileName.c_str()); - if (BufSize < (unsigned)St.st_size+1) + if (BufSize < (unsigned long long)St.st_size+1) { BufSize = St.st_size+1; Buffer = (char *)realloc(Buffer,St.st_size+1); @@ -1067,7 +1067,7 @@ void ReleaseWriter::Finish() for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.MD5.c_str(), (*I).second.size, (*I).first.c_str()); @@ -1079,7 +1079,7 @@ void ReleaseWriter::Finish() for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.SHA1.c_str(), (*I).second.size, (*I).first.c_str()); @@ -1091,7 +1091,7 @@ void ReleaseWriter::Finish() for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.SHA256.c_str(), (*I).second.size, (*I).first.c_str()); @@ -1103,7 +1103,7 @@ void ReleaseWriter::Finish() I != CheckSums.end(); ++I) { - fprintf(Output, " %s %32ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.SHA512.c_str(), (*I).second.size, (*I).first.c_str()); diff --git a/ftparchive/writer.h b/ftparchive/writer.h index c6026e954..a43b83876 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -48,7 +48,7 @@ class FTWScanner static int ScannerFile(const char *File, bool const &ReadLink); bool Delink(string &FileName,const char *OriginalPath, - unsigned long &Bytes,off_t const &FileSize); + unsigned long long &Bytes,unsigned long long const &FileSize); inline void NewLine(unsigned const &Priority) { @@ -159,7 +159,7 @@ class SourcesWriter : public FTWScanner Override BOver; Override SOver; char *Buffer; - unsigned long BufSize; + unsigned long long BufSize; public: @@ -198,7 +198,7 @@ protected: string SHA256; string SHA512; // Limited by FileFd::Size() - unsigned long size; + unsigned long long size; ~CheckSum() {}; }; map<string,struct CheckSum> CheckSums; diff --git a/methods/ftp.cc b/methods/ftp.cc index a445d767c..87aa8d798 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -629,7 +629,7 @@ bool FTPConn::ExtGoPasv() // FTPConn::Size - Return the size of a file /*{{{*/ // --------------------------------------------------------------------- /* Grab the file size from the server, 0 means no size or empty file */ -bool FTPConn::Size(const char *Path,unsigned long &Size) +bool FTPConn::Size(const char *Path,unsigned long long &Size) { // Query the size unsigned int Tag; @@ -639,7 +639,7 @@ bool FTPConn::Size(const char *Path,unsigned long &Size) return false; char *End; - Size = strtol(Msg.c_str(),&End,10); + Size = strtoull(Msg.c_str(),&End,10); if (Tag >= 400 || End == Msg.c_str()) Size = 0; return true; @@ -841,7 +841,7 @@ bool FTPConn::Finalize() // --------------------------------------------------------------------- /* This opens a data connection, sends REST and RETR and then transfers the file over. */ -bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume, +bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, Hashes &Hash,bool &Missing) { Missing = false; @@ -1004,7 +1004,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) // Get the files information Status(_("Query")); - unsigned long Size; + unsigned long long Size; if (Server->Size(File,Size) == false || Server->ModTime(File,FailTime) == false) { @@ -1026,7 +1026,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) struct stat Buf; if (stat(Itm->DestFile.c_str(),&Buf) == 0) { - if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime) + if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) { Res.Size = Buf.st_size; Res.LastModified = Buf.st_mtime; @@ -1036,7 +1036,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) } // Resume? - if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size) + if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size) Res.ResumePoint = Buf.st_size; } diff --git a/methods/ftp.h b/methods/ftp.h index d7f1f7fbe..b4913ca57 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -53,9 +53,9 @@ class FTPConn bool ExtGoPasv(); // Query - bool Size(const char *Path,unsigned long &Size); + bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); - bool Get(const char *Path,FileFd &To,unsigned long Resume, + bool Get(const char *Path,FileFd &To,unsigned long long Resume, Hashes &MD5,bool &Missing); FTPConn(URI Srv); diff --git a/methods/gzip.cc b/methods/gzip.cc index 6202d73dc..f4bb052e2 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -64,7 +64,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) while (1) { unsigned char Buffer[4*1024]; - unsigned long Count; + unsigned long long Count = 0; if (!From.Read(Buffer,sizeof(Buffer),&Count)) { diff --git a/methods/http.cc b/methods/http.cc index e505b816e..51fdaa0cd 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -65,15 +65,15 @@ bool AllowRedirect = false; bool Debug = false; URI Proxy; -unsigned long CircleBuf::BwReadLimit=0; -unsigned long CircleBuf::BwTickReadData=0; +unsigned long long CircleBuf::BwReadLimit=0; +unsigned long long CircleBuf::BwTickReadData=0; struct timeval CircleBuf::BwReadTick={0,0}; const unsigned int CircleBuf::BW_HZ=10; // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ -CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0) +CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0) { Buf = new unsigned char[Size]; Reset(); @@ -89,7 +89,7 @@ void CircleBuf::Reset() InP = 0; OutP = 0; StrPos = 0; - MaxGet = (unsigned int)-1; + MaxGet = (unsigned long long)-1; OutQueue = string(); if (Hash != 0) { @@ -104,7 +104,7 @@ void CircleBuf::Reset() is non-blocking.. */ bool CircleBuf::Read(int Fd) { - unsigned long BwReadMax; + unsigned long long BwReadMax; while (1) { @@ -119,7 +119,7 @@ bool CircleBuf::Read(int Fd) struct timeval now; gettimeofday(&now,0); - unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 + + unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 + now.tv_usec-CircleBuf::BwReadTick.tv_usec; if(d > 1000000/BW_HZ) { CircleBuf::BwReadTick = now; @@ -133,7 +133,7 @@ bool CircleBuf::Read(int Fd) } // Write the buffer segment - int Res; + ssize_t Res; if(CircleBuf::BwReadLimit) { Res = read(Fd,Buf + (InP%Size), BwReadMax > LeftRead() ? LeftRead() : BwReadMax); @@ -182,7 +182,7 @@ void CircleBuf::FillOut() return; // Write the buffer segment - unsigned long Sz = LeftRead(); + unsigned long long Sz = LeftRead(); if (OutQueue.length() - StrPos < Sz) Sz = OutQueue.length() - StrPos; memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz); @@ -216,7 +216,7 @@ bool CircleBuf::Write(int Fd) return true; // Write the buffer segment - int Res; + ssize_t Res; Res = write(Fd,Buf + (OutP%Size),LeftWrite()); if (Res == 0) @@ -242,7 +242,7 @@ bool CircleBuf::Write(int Fd) bool CircleBuf::WriteTillEl(string &Data,bool Single) { // We cheat and assume it is unneeded to have more than one buffer load - for (unsigned long I = OutP; I < InP; I++) + for (unsigned long long I = OutP; I < InP; I++) { if (Buf[I%Size] != '\n') continue; @@ -260,7 +260,7 @@ bool CircleBuf::WriteTillEl(string &Data,bool Single) Data = ""; while (OutP < I) { - unsigned long Sz = LeftWrite(); + unsigned long long Sz = LeftWrite(); if (Sz == 0) return false; if (I - OutP < Sz) @@ -455,7 +455,7 @@ bool ServerState::RunData() return false; // See if we are done - unsigned long Len = strtol(Data.c_str(),0,16); + unsigned long long Len = strtoull(Data.c_str(),0,16); if (Len == 0) { In.Limit(-1); @@ -598,7 +598,7 @@ bool ServerState::HeaderLine(string Line) if (StartPos != 0) return true; - if (sscanf(Val.c_str(),"%lu",&Size) != 1) + if (sscanf(Val.c_str(),"%llu",&Size) != 1) return _error->Error(_("The HTTP server sent an invalid Content-Length header")); return true; } @@ -613,9 +613,9 @@ bool ServerState::HeaderLine(string Line) { HaveContent = true; - if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2) + if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); - if ((unsigned)StartPos > Size) + if ((unsigned long long)StartPos > Size) return _error->Error(_("This HTTP server has broken range support")); return true; } @@ -718,7 +718,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out) if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { // In this case we send an if-range query with a range header - sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1, + sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size - 1, TimeRFC1123(SBuf.st_mtime).c_str()); Req += Buf; } diff --git a/methods/http.h b/methods/http.h index aa96c6810..b74740ab3 100644 --- a/methods/http.h +++ b/methods/http.h @@ -23,29 +23,29 @@ class HttpMethod; class CircleBuf { unsigned char *Buf; - unsigned long Size; - unsigned long InP; - unsigned long OutP; + unsigned long long Size; + unsigned long long InP; + unsigned long long OutP; string OutQueue; - unsigned long StrPos; - unsigned long MaxGet; + unsigned long long StrPos; + unsigned long long MaxGet; struct timeval Start; - static unsigned long BwReadLimit; - static unsigned long BwTickReadData; + static unsigned long long BwReadLimit; + static unsigned long long BwTickReadData; static struct timeval BwReadTick; static const unsigned int BW_HZ; - unsigned long LeftRead() + unsigned long long LeftRead() { - unsigned long Sz = Size - (InP - OutP); + unsigned long long Sz = Size - (InP - OutP); if (Sz > Size - (InP%Size)) Sz = Size - (InP%Size); return Sz; } - unsigned long LeftWrite() + unsigned long long LeftWrite() { - unsigned long Sz = InP - OutP; + unsigned long long Sz = InP - OutP; if (InP > MaxGet) Sz = MaxGet - OutP; if (Sz > Size - (OutP%Size)) @@ -67,7 +67,7 @@ class CircleBuf bool WriteTillEl(string &Data,bool Single = false); // Control the write limit - void Limit(long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;} + void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;} bool IsLimit() {return MaxGet == OutP;}; void Print() {cout << MaxGet << ',' << OutP << endl;}; @@ -79,7 +79,7 @@ class CircleBuf void Reset(); void Stats(); - CircleBuf(unsigned long Size); + CircleBuf(unsigned long long Size); ~CircleBuf() {delete [] Buf; delete Hash;}; }; @@ -92,8 +92,8 @@ struct ServerState char Code[MAXLEN]; // These are some statistics from the last parsed header lines - unsigned long Size; - signed long StartPos; + unsigned long long Size; + signed long long StartPos; time_t Date; bool HaveContent; enum {Chunked,Stream,Closes} Encoding; diff --git a/methods/https.cc b/methods/https.cc index 45bd2a367..06a0e285a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -52,7 +52,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow, { HttpsMethod *me = (HttpsMethod *)clientp; if(dltotal > 0 && me->Res.Size == 0) { - me->Res.Size = (unsigned long)dltotal; + me->Res.Size = (unsigned long long)dltotal; me->URIStart(me->Res); } return 0; diff --git a/methods/rred.cc b/methods/rred.cc index 80fc98ac5..57d9a8437 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -256,7 +256,7 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ #ifdef _POSIX_MAPPED_FILES MMap ed_cmds(MMap::ReadOnly); if (Patch.gzFd() != NULL) { - unsigned long mapSize = Patch.Size(); + unsigned long long mapSize = Patch.Size(); DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0); if (dyn->validData() == false) { delete dyn; diff --git a/methods/rsh.cc b/methods/rsh.cc index 10fe76dc3..c95a4d3eb 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -251,7 +251,7 @@ bool RSHConn::WriteMsg(string &Text,bool Sync,const char *Fmt,...) // --------------------------------------------------------------------- /* Right now for successfull transfer the file size must be known in advance. */ -bool RSHConn::Size(const char *Path,unsigned long &Size) +bool RSHConn::Size(const char *Path,unsigned long long &Size) { // Query the size string Msg; @@ -263,7 +263,7 @@ bool RSHConn::Size(const char *Path,unsigned long &Size) // FIXME: Sense if the bad reply is due to a File Not Found. char *End; - Size = strtoul(Msg.c_str(),&End,10); + Size = strtoull(Msg.c_str(),&End,10); if (End == Msg.c_str()) return _error->Error(_("File not found")); return true; @@ -288,8 +288,8 @@ bool RSHConn::ModTime(const char *Path, time_t &Time) // RSHConn::Get - Get a file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume, - Hashes &Hash,bool &Missing, unsigned long Size) +bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &Hash,bool &Missing, unsigned long long Size) { Missing = false; @@ -314,7 +314,7 @@ bool RSHConn::Get(const char *Path,FileFd &To,unsigned long Resume, return false; // Copy loop - unsigned int MyLen = Resume; + unsigned long long MyLen = Resume; unsigned char Buffer[4096]; while (MyLen < Size) { @@ -428,7 +428,7 @@ bool RSHMethod::Fetch(FetchItem *Itm) Status(_("Connecting to %s"), Get.Host.c_str()); // Get the files information - unsigned long Size; + unsigned long long Size; if (Server->Size(File,Size) == false || Server->ModTime(File,FailTime) == false) { @@ -449,7 +449,7 @@ bool RSHMethod::Fetch(FetchItem *Itm) // See if the file exists struct stat Buf; if (stat(Itm->DestFile.c_str(),&Buf) == 0) { - if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime) { + if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime) { Res.Size = Buf.st_size; Res.LastModified = Buf.st_mtime; Res.ResumePoint = Buf.st_size; @@ -458,7 +458,7 @@ bool RSHMethod::Fetch(FetchItem *Itm) } // Resume? - if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size) + if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size) Res.ResumePoint = Buf.st_size; } diff --git a/methods/rsh.h b/methods/rsh.h index b06d5a94e..b4c76479a 100644 --- a/methods/rsh.h +++ b/methods/rsh.h @@ -41,10 +41,10 @@ class RSHConn void Close(); // Query - bool Size(const char *Path,unsigned long &Size); + bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); - bool Get(const char *Path,FileFd &To,unsigned long Resume, - Hashes &Hash,bool &Missing, unsigned long Size); + bool Get(const char *Path,FileFd &To,unsigned long long Resume, + Hashes &Hash,bool &Missing, unsigned long long Size); RSHConn(URI Srv); ~RSHConn(); |